From a383c2be68c8ef584a8d33c35f4e719c678b5a4b Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 1 Apr 2025 18:05:59 -0700 Subject: [PATCH 01/18] HDDS-12501. Remove OMKeyInfo from SST files in backup directory. --- .../rocksdiff/RocksDBCheckpointDiffer.java | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index e57c501640d6..0f11d8dddeb1 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -38,6 +39,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; @@ -65,10 +67,14 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.Scheduler; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileIterator; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader; +import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.ozone.compaction.log.CompactionFileInfo; import org.apache.ozone.compaction.log.CompactionLogEntry; @@ -237,8 +243,13 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, this::pruneSstFiles, pruneCompactionDagDaemonRunIntervalInMs, pruneCompactionDagDaemonRunIntervalInMs, - TimeUnit.MILLISECONDS - ); + TimeUnit.MILLISECONDS); + + this.scheduler.scheduleWithFixedDelay( + this::pruneSstFileValues, + pruneCompactionDagDaemonRunIntervalInMs, + pruneCompactionDagDaemonRunIntervalInMs, + TimeUnit.MILLISECONDS); } else { this.scheduler = null; } @@ -1382,6 +1393,62 @@ public void pruneSstFiles() { } } + /** + * Defines the task that removes OMKeyInfo from SST files from backup directory to + * save disk space. + */ + public void pruneSstFileValues() { + if (!shouldRun()) { + return; + } + Path sstBackupDirPath = Paths.get(sstBackupDir); + String kVSeparator = ","; + + try (Stream files = Files.list(sstBackupDirPath) + .filter(file -> file.endsWith(ROCKSDB_SST_SUFFIX)) ) { + + for (Path file : files.collect(Collectors.toList())) { + // Write the file.sst => file.sst.tmp + File sstFile = file.toFile(); + File prunedSSTFile = Files.createFile(sstBackupDirPath + .resolve( sstFile.getName() + ".tmp")).toFile(); + + ManagedEnvOptions envOptions = new ManagedEnvOptions(); + ManagedOptions managedOptions = new ManagedOptions(); + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) + sstFileWriter.open(prunedSSTFile.getAbsolutePath()); + + ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( + managedOptions, sstFile.getAbsolutePath(), 2 * 1024 * 1024); + ManagedRawSSTFileIterator itr = sstFileReader.newIterator( + keyValue -> StringUtils.bytes2String(keyValue.getKey()) + kVSeparator + + StringUtils.bytes2String(keyValue.getValue()), null, null); + + while(itr.hasNext()) { + String[] keyValue = itr.next().split(kVSeparator); + if (keyValue[1].isEmpty()) { + sstFileWriter.delete(keyValue[0].getBytes(UTF_8)); + } else { + sstFileWriter.put(keyValue[0].getBytes(UTF_8), "\0".getBytes(UTF_8)); + } + } + sstFileWriter.finish(); + + // Acquire a mutex + try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { + // Move file.sst.tmp file.sst and replace existing file atomically + Files.move(prunedSSTFile.toPath(), file, StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + LOG.debug("Prune OMKeyInfo from SST files: {}", sstFile.getPath()); + } + } catch (IOException e) { + LOG.error("Could not prune OMKeyInfo from SST files.", e); + } + } + public boolean shouldRun() { return !suspended.get(); } From 37edeebc913b1b735ba7ef1102a2a4e97477ee91 Mon Sep 17 00:00:00 2001 From: saketa Date: Mon, 7 Apr 2025 14:02:09 -0700 Subject: [PATCH 02/18] HDDS-12501. Added Read/Write SSt File lock. --- .../rocksdiff/RocksDBCheckpointDiffer.java | 72 +++++++++++++------ 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 0f11d8dddeb1..3f8e8196febf 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -18,6 +18,10 @@ package org.apache.ozone.rocksdiff; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_FAIR_LOCK; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_FAIR_LOCK_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_STRIPED_LOCK_SIZE_PREFIX; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_STRIPED_LOCK_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; @@ -29,6 +33,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.graph.GraphBuilder; import com.google.common.graph.MutableGraph; +import com.google.common.util.concurrent.Striped; import com.google.protobuf.InvalidProtocolBufferException; import java.io.BufferedWriter; import java.io.File; @@ -56,6 +61,9 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.collections.CollectionUtils; @@ -66,6 +74,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.Scheduler; +import org.apache.hadoop.hdds.utils.SimpleStriped; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; @@ -169,6 +178,8 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final long maxAllowedTimeInDag; private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock(); + private final Striped stripedSSTLock; + private static final String SST_FILE_LOCK = "SST_FILE_LOCK"; private ColumnFamilyHandle snapshotInfoTableCFHandle; private static final String DAG_PRUNING_SERVICE_NAME = "CompactionDagPruningService"; @@ -223,6 +234,13 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, TimeUnit.MILLISECONDS); this.suspended = new AtomicBoolean(false); + this.stripedSSTLock = SimpleStriped.readWriteLock( + configuration.getInt(OZONE_MANAGER_STRIPED_LOCK_SIZE_PREFIX + + SST_FILE_LOCK.toLowerCase(), + OZONE_MANAGER_STRIPED_LOCK_SIZE_DEFAULT), + configuration.getBoolean(OZONE_MANAGER_FAIR_LOCK, + OZONE_MANAGER_FAIR_LOCK_DEFAULT)); + long pruneCompactionDagDaemonRunIntervalInMs = configuration.getTimeDuration( OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, @@ -854,9 +872,15 @@ public synchronized Optional> getSSTDiffListWithFullPath(DifferSnap String sstFullPath = getSSTFullPath(sst, src.getDbPath(), dest.getDbPath()); Path link = Paths.get(sstFilesDirForSnapDiffJob, sst + SST_FILE_EXTENSION); - Path srcFile = Paths.get(sstFullPath); - createLink(link, srcFile); - return link.toString(); + // Take a read lock on the SST FILE + getSSTFileLock(link.toFile().getAbsolutePath()).readLock().lock(); + try { + Path srcFile = Paths.get(sstFullPath); + createLink(link, srcFile); + return link.toString(); + } finally { + getSSTFileLock(link.toFile().getAbsolutePath()).readLock().unlock(); + } }) .collect(Collectors.toList())); } @@ -1402,49 +1426,49 @@ public void pruneSstFileValues() { return; } Path sstBackupDirPath = Paths.get(sstBackupDir); - String kVSeparator = ","; try (Stream files = Files.list(sstBackupDirPath) - .filter(file -> file.endsWith(ROCKSDB_SST_SUFFIX)) ) { - + .filter(file -> file.endsWith(ROCKSDB_SST_SUFFIX))) { + ManagedEnvOptions envOptions = new ManagedEnvOptions(); + ManagedOptions managedOptions = new ManagedOptions(); for (Path file : files.collect(Collectors.toList())) { // Write the file.sst => file.sst.tmp File sstFile = file.toFile(); File prunedSSTFile = Files.createFile(sstBackupDirPath .resolve( sstFile.getName() + ".tmp")).toFile(); - ManagedEnvOptions envOptions = new ManagedEnvOptions(); - ManagedOptions managedOptions = new ManagedOptions(); - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions); sstFileWriter.open(prunedSSTFile.getAbsolutePath()); ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( managedOptions, sstFile.getAbsolutePath(), 2 * 1024 * 1024); - ManagedRawSSTFileIterator itr = sstFileReader.newIterator( - keyValue -> StringUtils.bytes2String(keyValue.getKey()) + kVSeparator - + StringUtils.bytes2String(keyValue.getValue()), null, null); + Function> keyValueFuntion = + keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()); + ManagedRawSSTFileIterator> itr = sstFileReader.newIterator(keyValueFuntion, null, null); while(itr.hasNext()) { - String[] keyValue = itr.next().split(kVSeparator); - if (keyValue[1].isEmpty()) { - sstFileWriter.delete(keyValue[0].getBytes(UTF_8)); + Pair keyValue = itr.next(); + if (keyValue.getValue() == 0) { + sstFileWriter.delete(keyValue.getKey()); } else { - sstFileWriter.put(keyValue[0].getBytes(UTF_8), "\0".getBytes(UTF_8)); + sstFileWriter.put(keyValue.getKey(), new byte[0]); } } sstFileWriter.finish(); - // Acquire a mutex - try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { + // Take a write lock on the SST File + getSSTFileLock(sstFile.getAbsolutePath()).writeLock().lock(); + try { // Move file.sst.tmp file.sst and replace existing file atomically - Files.move(prunedSSTFile.toPath(), file, StandardCopyOption.ATOMIC_MOVE, - StandardCopyOption.REPLACE_EXISTING); - } catch (InterruptedException e) { + Files.move(prunedSSTFile.toPath(), file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { throw new RuntimeException(e); + } finally { + getSSTFileLock(sstFile.getAbsolutePath()).writeLock().unlock(); } LOG.debug("Prune OMKeyInfo from SST files: {}", sstFile.getPath()); } - } catch (IOException e) { + } catch (Exception e) { LOG.error("Could not prune OMKeyInfo from SST files.", e); } } @@ -1513,6 +1537,10 @@ public BootstrapStateHandler.Lock getBootstrapStateLock() { return lock; } + public ReentrantReadWriteLock getSSTFileLock(String key) { + return (ReentrantReadWriteLock) stripedSSTLock.get(key); + } + public void pngPrintMutableGraph(String filePath, GraphType graphType) throws IOException { Objects.requireNonNull(filePath, "Image file path is required."); From 73e0d2a38399ddb004426e2276c6ff024bc49220 Mon Sep 17 00:00:00 2001 From: saketa Date: Fri, 18 Apr 2025 14:55:49 -0700 Subject: [PATCH 03/18] HDDS-12501. Used BlockingQueue to prune backup SST files on compaction complete. Using SST lock from OM. --- .../ozone/lock/StripedLockProvider.java | 28 +++ .../hadoop/hdds/utils/db/DBStoreBuilder.java | 8 +- .../apache/hadoop/hdds/utils/db/RDBStore.java | 7 +- .../hadoop/hdds/utils/db/TestRDBStore.java | 2 +- .../src/main/proto/hdds.proto | 1 + .../compaction/log/CompactionFileInfo.java | 34 ++- .../ozone/rocksdiff/CompactionNode.java | 18 +- .../rocksdiff/RocksDBCheckpointDiffer.java | 215 ++++++++++++------ .../TestRocksDBCheckpointDiffer.java | 2 +- .../ozone/om/lock/IOzoneManagerLock.java | 3 +- .../hadoop/ozone/om/lock/OmReadOnlyLock.java | 8 + .../ozone/om/lock/OzoneManagerLock.java | 8 +- .../ozone/om/OmMetadataManagerImpl.java | 17 +- .../hadoop/ozone/om/TestOMDBDefinition.java | 4 +- .../hadoop/ozone/repair/om/FSORepairTool.java | 5 +- 15 files changed, 274 insertions(+), 86 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/StripedLockProvider.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/StripedLockProvider.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/StripedLockProvider.java new file mode 100644 index 000000000000..a05fa4d2b485 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/StripedLockProvider.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.lock; + +import com.google.common.util.concurrent.Striped; +import java.util.concurrent.locks.ReadWriteLock; + +/** + * Interface for locks. + */ +public interface StripedLockProvider { + Striped getStripedLock(String key); +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index c6f9e1c7cf1b..e17ee41733a0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedStatistics; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions; +import org.apache.hadoop.ozone.lock.StripedLockProvider; import org.eclipse.jetty.util.StringUtil; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.InfoLogLevel; @@ -106,6 +107,7 @@ public final class DBStoreBuilder { // number in request to avoid increase in heap memory. private long maxDbUpdatesSizeThreshold; private Integer maxNumberOfOpenFiles = null; + private StripedLockProvider lock = null; /** * Create DBStoreBuilder from a generic DBDefinition. @@ -231,7 +233,7 @@ public DBStore build() throws IOException { return new RDBStore(dbFile, rocksDBOption, statistics, writeOptions, tableConfigs, registry.build(), openReadOnly, dbJmxBeanNameName, enableCompactionDag, maxDbUpdatesSizeThreshold, createCheckpointDirs, configuration, - enableRocksDbMetrics); + enableRocksDbMetrics, lock); } finally { tableConfigs.forEach(TableConfig::close); } @@ -302,6 +304,10 @@ public DBStoreBuilder setEnableRocksDbMetrics(boolean enableRocksDbMetrics) { this.enableRocksDbMetrics = enableRocksDbMetrics; return this; } + public DBStoreBuilder setLock(StripedLockProvider lock) { + this.lock = lock; + return this; + } /** * Set the {@link ManagedDBOptions} and default * {@link ManagedColumnFamilyOptions} based on {@code prof}. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 0e7ee39db0a6..e3363190e8c7 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedStatistics; import org.apache.hadoop.hdds.utils.db.managed.ManagedTransactionLogIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions; +import org.apache.hadoop.ozone.lock.StripedLockProvider; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.RocksDBCheckpointDifferHolder; import org.rocksdb.RocksDBException; @@ -83,7 +84,8 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, ManagedStatistics stati long maxDbUpdatesSizeThreshold, boolean createCheckpointDirs, ConfigurationSource configuration, - boolean enableRocksDBMetrics) + boolean enableRocksDBMetrics, + StripedLockProvider lock) throws IOException { Preconditions.checkNotNull(dbFile, "DB file location cannot be null"); @@ -103,7 +105,8 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, ManagedStatistics stati DB_COMPACTION_SST_BACKUP_DIR, DB_COMPACTION_LOG_DIR, dbLocation.toString(), - configuration); + configuration, + lock); rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions); } else { rocksDBCheckpointDiffer = null; diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java index f8c14143e211..8f323c77c16b 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java @@ -78,7 +78,7 @@ public static RDBStore newRDBStore(File dbFile, ManagedDBOptions options, throws IOException { return new RDBStore(dbFile, options, null, new ManagedWriteOptions(), families, CodecRegistry.newBuilder().build(), false, null, false, - maxDbUpdatesSizeThreshold, true, null, true); + maxDbUpdatesSizeThreshold, true, null, true, null); } public static final int MAX_DB_UPDATES_SIZE_THRESHOLD = 80; diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index 45dd22e49fe1..c1e3fe034f26 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -515,6 +515,7 @@ message CompactionFileInfoProto { optional string startKey = 2; optional string endKey = 3; optional string columnFamily = 4; + optional bool pruned = 5; } message CompactionLogEntryProto { diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java index 5ad8513cfb40..46189600bb44 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java @@ -32,16 +32,26 @@ public final class CompactionFileInfo { private final String startKey; private final String endKey; private final String columnFamily; + private boolean pruned; @VisibleForTesting public CompactionFileInfo(String fileName, String startRange, String endRange, String columnFamily) { + this(fileName, startRange, endRange, columnFamily, false); + } + + public CompactionFileInfo(String fileName, + String startRange, + String endRange, + String columnFamily, + boolean pruned) { this.fileName = fileName; this.startKey = startRange; this.endKey = endRange; this.columnFamily = columnFamily; + this.pruned = pruned; } public String getFileName() { @@ -60,10 +70,19 @@ public String getColumnFamily() { return columnFamily; } + public boolean isPruned() { + return pruned; + } + + public void setPruned() { + this.pruned = true; + } + public HddsProtos.CompactionFileInfoProto getProtobuf() { HddsProtos.CompactionFileInfoProto.Builder builder = HddsProtos.CompactionFileInfoProto.newBuilder() - .setFileName(fileName); + .setFileName(fileName) + .setPruned(pruned); if (startKey != null) { builder = builder.setStartKey(startKey); } @@ -89,6 +108,9 @@ public static CompactionFileInfo getFromProtobuf( if (proto.hasColumnFamily()) { builder.setColumnFamily(proto.getColumnFamily()); } + if (proto.hasPruned() && proto.getPruned()) { + builder.setPruned(); + } return builder.build(); } @@ -96,7 +118,7 @@ public static CompactionFileInfo getFromProtobuf( @Override public String toString() { return String.format("fileName: '%s', startKey: '%s', endKey: '%s'," + - " columnFamily: '%s'", fileName, startKey, endKey, columnFamily); + " columnFamily: '%s', isPruned: '%b'", fileName, startKey, endKey, columnFamily, pruned); } /** @@ -107,6 +129,7 @@ public static class Builder { private String startRange; private String endRange; private String columnFamily; + private boolean pruned = false; public Builder(String fileName) { Preconditions.checkNotNull(fileName, "FileName is required parameter."); @@ -138,6 +161,11 @@ public Builder setValues(LiveFileMetaData fileMetaData) { return this; } + public Builder setPruned() { + this.pruned = true; + return this; + } + public CompactionFileInfo build() { if ((startRange != null || endRange != null || columnFamily != null) && (startRange == null || endRange == null || columnFamily == null)) { @@ -149,7 +177,7 @@ public CompactionFileInfo build() { } return new CompactionFileInfo(fileName, startRange, endRange, - columnFamily); + columnFamily, pruned); } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java index 4e7c38c62c9a..b7c15a899dc3 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java @@ -31,6 +31,7 @@ public class CompactionNode { private final String startKey; private final String endKey; private final String columnFamily; + private boolean pruned; /** * CompactionNode constructor. @@ -41,6 +42,12 @@ public class CompactionNode { public CompactionNode(String file, long numKeys, long seqNum, String startKey, String endKey, String columnFamily) { + this(file, numKeys, seqNum, startKey, endKey, columnFamily, false); + } + + public CompactionNode(String file, long numKeys, long seqNum, + String startKey, String endKey, String columnFamily, + boolean pruned) { fileName = file; totalNumberOfKeys = numKeys; snapshotGeneration = seqNum; @@ -48,11 +55,12 @@ public CompactionNode(String file, long numKeys, long seqNum, this.startKey = startKey; this.endKey = endKey; this.columnFamily = columnFamily; + this.pruned = pruned; } public CompactionNode(CompactionFileInfo compactionFileInfo) { this(compactionFileInfo.getFileName(), -1, -1, compactionFileInfo.getStartKey(), - compactionFileInfo.getEndKey(), compactionFileInfo.getColumnFamily()); + compactionFileInfo.getEndKey(), compactionFileInfo.getColumnFamily(), compactionFileInfo.isPruned()); } @Override @@ -88,6 +96,10 @@ public String getColumnFamily() { return columnFamily; } + public boolean isPruned() { + return pruned; + } + public void setCumulativeKeysReverseTraversal( long cumulativeKeysReverseTraversal) { this.cumulativeKeysReverseTraversal = cumulativeKeysReverseTraversal; @@ -96,4 +108,8 @@ public void setCumulativeKeysReverseTraversal( public void addCumulativeKeysReverseTraversal(long diff) { this.cumulativeKeysReverseTraversal += diff; } + + public void setPruned() { + this.pruned = true; + } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 3f8e8196febf..c5de678641c2 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -18,10 +18,6 @@ package org.apache.ozone.rocksdiff; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_FAIR_LOCK; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_FAIR_LOCK_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_STRIPED_LOCK_SIZE_PREFIX; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_STRIPED_LOCK_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; @@ -33,7 +29,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.graph.GraphBuilder; import com.google.common.graph.MutableGraph; -import com.google.common.util.concurrent.Striped; import com.google.protobuf.InvalidProtocolBufferException; import java.io.BufferedWriter; import java.io.File; @@ -57,13 +52,13 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.collections.CollectionUtils; @@ -73,8 +68,8 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.apache.hadoop.hdds.utils.Scheduler; -import org.apache.hadoop.hdds.utils.SimpleStriped; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; @@ -85,6 +80,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; +import org.apache.hadoop.ozone.lock.StripedLockProvider; import org.apache.ozone.compaction.log.CompactionFileInfo; import org.apache.ozone.compaction.log.CompactionLogEntry; import org.apache.ozone.graph.PrintableGraph; @@ -178,8 +174,9 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final long maxAllowedTimeInDag; private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock(); - private final Striped stripedSSTLock; + private final StripedLockProvider omLock; private static final String SST_FILE_LOCK = "SST_FILE_LOCK"; + private static final int SST_READ_AHEAD_SIZE = 2 * 1024 * 1024; private ColumnFamilyHandle snapshotInfoTableCFHandle; private static final String DAG_PRUNING_SERVICE_NAME = "CompactionDagPruningService"; @@ -188,6 +185,9 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private ColumnFamilyHandle compactionLogTableCFHandle; private ManagedRocksDB activeRocksDB; private ConcurrentMap inflightCompactions; + private boolean isNativeLibsLoaded; + private BlockingQueue> pruneQueue = + new LinkedBlockingQueue>(); /** * For snapshot diff calculation we only need to track following column @@ -214,7 +214,8 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, String sstBackupDirName, String compactionLogDirName, String activeDBLocationName, - ConfigurationSource configuration) { + ConfigurationSource configuration, + StripedLockProvider omLock) { Preconditions.checkNotNull(metadataDirName); Preconditions.checkNotNull(sstBackupDirName); Preconditions.checkNotNull(compactionLogDirName); @@ -225,6 +226,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, createCompactionLogDir(metadataDirName, compactionLogDirName); this.sstBackupDir = Paths.get(metadataDirName, sstBackupDirName) + "/"; createSstBackUpDir(); + this.omLock = omLock; // Active DB location is used in getSSTFileSummary this.activeDBLocationStr = activeDBLocationName + "/"; @@ -234,13 +236,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, TimeUnit.MILLISECONDS); this.suspended = new AtomicBoolean(false); - this.stripedSSTLock = SimpleStriped.readWriteLock( - configuration.getInt(OZONE_MANAGER_STRIPED_LOCK_SIZE_PREFIX + - SST_FILE_LOCK.toLowerCase(), - OZONE_MANAGER_STRIPED_LOCK_SIZE_DEFAULT), - configuration.getBoolean(OZONE_MANAGER_FAIR_LOCK, - OZONE_MANAGER_FAIR_LOCK_DEFAULT)); - long pruneCompactionDagDaemonRunIntervalInMs = configuration.getTimeDuration( OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, @@ -263,11 +258,20 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, pruneCompactionDagDaemonRunIntervalInMs, TimeUnit.MILLISECONDS); - this.scheduler.scheduleWithFixedDelay( - this::pruneSstFileValues, - pruneCompactionDagDaemonRunIntervalInMs, - pruneCompactionDagDaemonRunIntervalInMs, - TimeUnit.MILLISECONDS); + try { + isNativeLibsLoaded = ManagedRawSSTFileReader.loadLibrary(); + } catch (NativeLibraryNotLoadedException e) { + LOG.warn("Native Library for raw sst file reading loading failed." + + " Cannot prune OMKeyInfo from SST files. {}", e.getMessage()); + isNativeLibsLoaded = false; + } + if (isNativeLibsLoaded) { + this.scheduler.scheduleWithFixedDelay( + this::pruneSstFileValues, + pruneCompactionDagDaemonRunIntervalInMs, + pruneCompactionDagDaemonRunIntervalInMs, + TimeUnit.MILLISECONDS); + } } else { this.scheduler = null; } @@ -533,6 +537,7 @@ public void onCompactionCompleted(RocksDB db, } CompactionLogEntry compactionLogEntry = builder.build(); + byte[] key; synchronized (this) { if (closed) { return; @@ -545,7 +550,7 @@ public void onCompactionCompleted(RocksDB db, } // Add the compaction log entry to Compaction log table. - addToCompactionLogTable(compactionLogEntry); + key = addToCompactionLogTable(compactionLogEntry); // Populate the DAG populateCompactionDAG(compactionLogEntry.getInputFileInfoList(), @@ -555,12 +560,23 @@ public void onCompactionCompleted(RocksDB db, inflightCompactions.remove(inputFile); } } + // Add the compaction log entry to the prune queue + // so that the backup input sst files can be pruned. + if (isNativeLibsLoaded) { + try { + LOG.info("OnCompactionCompleted: {}", compactionLogEntry); + pruneQueue.put(Pair.of(key, compactionLogEntry)); + } catch (InterruptedException e) { + LOG.error("Could not add backup SST files in queue to be pruned. {}", + compactionLogEntry.getInputFileInfoList(), e); + } + } } }; } @VisibleForTesting - void addToCompactionLogTable(CompactionLogEntry compactionLogEntry) { + byte[] addToCompactionLogTable(CompactionLogEntry compactionLogEntry) { String dbSequenceIdStr = String.valueOf(compactionLogEntry.getDbSequenceNumber()); @@ -585,6 +601,7 @@ void addToCompactionLogTable(CompactionLogEntry compactionLogEntry) { // TODO: Revisit exception handling before merging the PR. throw new RuntimeException(exception); } + return key; } /** @@ -873,13 +890,14 @@ public synchronized Optional> getSSTDiffListWithFullPath(DifferSnap Path link = Paths.get(sstFilesDirForSnapDiffJob, sst + SST_FILE_EXTENSION); // Take a read lock on the SST FILE - getSSTFileLock(link.toFile().getAbsolutePath()).readLock().lock(); + ReentrantReadWriteLock.ReadLock sstReadLock = getSSTFileLock(sstFullPath).readLock(); + sstReadLock.lock(); try { Path srcFile = Paths.get(sstFullPath); createLink(link, srcFile); return link.toString(); } finally { - getSSTFileLock(link.toFile().getAbsolutePath()).readLock().unlock(); + sstReadLock.unlock(); } }) .collect(Collectors.toList())); @@ -1175,7 +1193,17 @@ private void addFileInfoToCompactionLogTable( builder.setCompactionReason(compactionReason); } - addToCompactionLogTable(builder.build()); + CompactionLogEntry compactionLogEntry = builder.build(); + byte[] key = addToCompactionLogTable(compactionLogEntry); + // Add the compaction log entry to the prune queue so that the backup input sst files can be pruned. + if (isNativeLibsLoaded) { + try { + pruneQueue.put(Pair.of(key, compactionLogEntry)); + } catch (InterruptedException e) { + LOG.error("Could not add backup SST files in queue to be pruned. {}", + compactionLogEntry.getInputFileInfoList(), e); + } + } } /** @@ -1426,47 +1454,98 @@ public void pruneSstFileValues() { return; } Path sstBackupDirPath = Paths.get(sstBackupDir); + try (ManagedEnvOptions envOptions = new ManagedEnvOptions(); + ManagedOptions managedOptions = new ManagedOptions(); + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { + + Pair compactionLogTableEntry; + while ((compactionLogTableEntry = pruneQueue.poll()) != null) { + + byte[] key = compactionLogTableEntry.getKey(); + CompactionLogEntry compactionLogEntry = compactionLogTableEntry.getValue(); + boolean shouldUpdateTable = false; + List fileInfoList = compactionLogEntry.getInputFileInfoList(); + List updatedFileInfoList = new ArrayList<>(); + for (CompactionFileInfo fileInfo : fileInfoList) { + if (fileInfo.isPruned()) { + updatedFileInfoList.add(fileInfo); + continue; + } + String sstFileName = fileInfo.getFileName(); + if (compactionNodeMap.get(sstFileName).isPruned()) { + shouldUpdateTable = true; + fileInfo.setPruned(); + updatedFileInfoList.add(fileInfo); + continue; + } + File sstFile = sstBackupDirPath.resolve(sstFileName + ROCKSDB_SST_SUFFIX).toFile(); + if (Files.notExists(sstFile.toPath())) { + LOG.debug("Skipping pruning SST file {} as it does not exist in backup directory.", sstFile); + updatedFileInfoList.add(fileInfo); + continue; + } - try (Stream files = Files.list(sstBackupDirPath) - .filter(file -> file.endsWith(ROCKSDB_SST_SUFFIX))) { - ManagedEnvOptions envOptions = new ManagedEnvOptions(); - ManagedOptions managedOptions = new ManagedOptions(); - for (Path file : files.collect(Collectors.toList())) { - // Write the file.sst => file.sst.tmp - File sstFile = file.toFile(); - File prunedSSTFile = Files.createFile(sstBackupDirPath - .resolve( sstFile.getName() + ".tmp")).toFile(); - - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions); - sstFileWriter.open(prunedSSTFile.getAbsolutePath()); - - ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( - managedOptions, sstFile.getAbsolutePath(), 2 * 1024 * 1024); - Function> keyValueFuntion = - keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()); - ManagedRawSSTFileIterator> itr = sstFileReader.newIterator(keyValueFuntion, null, null); - - while(itr.hasNext()) { - Pair keyValue = itr.next(); - if (keyValue.getValue() == 0) { - sstFileWriter.delete(keyValue.getKey()); - } else { - sstFileWriter.put(keyValue.getKey(), new byte[0]); + // Write the file.sst => file.sst.tmp + File prunedSSTFile = Files.createFile(sstBackupDirPath.resolve(sstFile.getName() + ".tmp")).toFile(); + ReentrantReadWriteLock.WriteLock sstWriteLock = getSSTFileLock(sstFile.getAbsolutePath()).writeLock(); + try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( + managedOptions, sstFile.getAbsolutePath(), SST_READ_AHEAD_SIZE); + ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( + keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null)) { + + sstFileWriter.open(prunedSSTFile.getAbsolutePath()); + while (itr.hasNext()) { + Pair keyValue = itr.next(); + if (keyValue.getValue() == 0) { + sstFileWriter.delete(keyValue.getKey()); + } else { + sstFileWriter.put(keyValue.getKey(), new byte[0]); + } + } + sstFileWriter.finish(); + + // Take a write lock on the SST File + sstWriteLock.lock(); + // Move file.sst.tmp to file.sst and replace existing file atomically + Files.move(prunedSSTFile.toPath(), sstFile.toPath(), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { + pruneQueue.put(compactionLogTableEntry); + LOG.error("Could not prune OMKeyInfo from {}. Reprocessing.", sstFileName, e); + continue; + } finally { + Files.deleteIfExists(prunedSSTFile.toPath()); + if (sstWriteLock.isHeldByCurrentThread()) { + sstWriteLock.unlock(); + } } + + // Put pruned flag in compaction DAG. Update CompactionNode in Map. + compactionNodeMap.get(sstFileName).setPruned(); + shouldUpdateTable = true; + fileInfo.setPruned(); + updatedFileInfoList.add(fileInfo); } - sstFileWriter.finish(); - - // Take a write lock on the SST File - getSSTFileLock(sstFile.getAbsolutePath()).writeLock().lock(); - try { - // Move file.sst.tmp file.sst and replace existing file atomically - Files.move(prunedSSTFile.toPath(), file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - getSSTFileLock(sstFile.getAbsolutePath()).writeLock().unlock(); + + if (shouldUpdateTable) { + // Update Compaction Log table. Track keys that need updating. + CompactionLogEntry.Builder builder = new CompactionLogEntry.Builder( + compactionLogEntry.getDbSequenceNumber(), + compactionLogEntry.getCompactionTime(), + updatedFileInfoList, + compactionLogEntry.getOutputFileInfoList()); + String compactionReason = compactionLogEntry.getCompactionReason(); + if (compactionReason != null) { + builder.setCompactionReason(compactionReason); + } + try { + activeRocksDB.get().put(compactionLogTableCFHandle, key, builder.build().getProtobuf().toByteArray()); + } catch (RocksDBException e) { + pruneQueue.put(compactionLogTableEntry); + LOG.error("Could not update pruned flag in compaction log table. Reprocessing entry: {}", + compactionLogEntry, e); + } } - LOG.debug("Prune OMKeyInfo from SST files: {}", sstFile.getPath()); } } catch (Exception e) { LOG.error("Could not prune OMKeyInfo from SST files.", e); @@ -1512,14 +1591,16 @@ public static RocksDBCheckpointDiffer getInstance( String sstBackupDirName, String compactionLogDirName, String activeDBLocationName, - ConfigurationSource configuration + ConfigurationSource configuration, + StripedLockProvider omLock ) { return INSTANCE_MAP.computeIfAbsent(metadataDirName, (key) -> new RocksDBCheckpointDiffer(metadataDirName, sstBackupDirName, compactionLogDirName, activeDBLocationName, - configuration)); + configuration, + omLock)); } /** @@ -1538,7 +1619,7 @@ public BootstrapStateHandler.Lock getBootstrapStateLock() { } public ReentrantReadWriteLock getSSTFileLock(String key) { - return (ReentrantReadWriteLock) stripedSSTLock.get(key); + return (ReentrantReadWriteLock) omLock.getStripedLock(SST_FILE_LOCK).get(key); } public void pngPrintMutableGraph(String filePath, GraphType graphType) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index c09c5af85d0c..603ae31a678b 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -188,7 +188,7 @@ public void init() throws RocksDBException { SST_BACK_UP_DIR_NAME, COMPACTION_LOG_DIR_NAME, ACTIVE_DB_DIR_NAME, - config); + config, null); ManagedColumnFamilyOptions cfOpts = new ManagedColumnFamilyOptions(); cfOpts.optimizeUniversalStyleCompaction(); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java index 6926b7d9bf23..c8beeb3df3a3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java @@ -19,11 +19,12 @@ import com.google.common.annotations.VisibleForTesting; import java.util.Collection; +import org.apache.hadoop.ozone.lock.StripedLockProvider; /** * Interface for OM Metadata locks. */ -public interface IOzoneManagerLock { +public interface IOzoneManagerLock extends StripedLockProvider { OMLockDetails acquireReadLock(OzoneManagerLock.Resource resource, String... resources); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java index 059536fe0a58..b1c23d699e74 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java @@ -20,7 +20,9 @@ import static org.apache.hadoop.ozone.om.lock.OMLockDetails.EMPTY_DETAILS_LOCK_ACQUIRED; import static org.apache.hadoop.ozone.om.lock.OMLockDetails.EMPTY_DETAILS_LOCK_NOT_ACQUIRED; +import com.google.common.util.concurrent.Striped; import java.util.Collection; +import java.util.concurrent.locks.ReadWriteLock; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource; /** @@ -108,4 +110,10 @@ public OMLockMetrics getOMLockMetrics() { throw new UnsupportedOperationException( "OmReadOnlyLock does not support this operation."); } + + @Override + public Striped getStripedLock(String key) { + throw new UnsupportedOperationException( + "OmReadOnlyLock does not support this operation."); + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java index 9fd567344cd0..2ead0da33a0c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java @@ -144,6 +144,11 @@ private ReentrantReadWriteLock getLock(Resource resource, String... keys) { return (ReentrantReadWriteLock) striped.get(key); } + @Override + public Striped getStripedLock(String resource) { + return stripedLockByResource.get(Resource.valueOf(resource)); + } + /** * Acquire read lock on resource. * @@ -586,7 +591,8 @@ public enum Resource { S3_SECRET_LOCK((byte) 4, "S3_SECRET_LOCK"), // 31 KEY_PATH_LOCK((byte) 5, "KEY_PATH_LOCK"), //63 PREFIX_LOCK((byte) 6, "PREFIX_LOCK"), //127 - SNAPSHOT_LOCK((byte) 7, "SNAPSHOT_LOCK"); // = 255 + SNAPSHOT_LOCK((byte) 7, "SNAPSHOT_LOCK"), // = 255 + SST_FILE_LOCK((byte) 8, "SST_FILE_LOCK"); // = 511 // level of the resource private byte lockLevel; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index d5fe33958aab..c5287c5642f3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -393,7 +393,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) int maxOpenFiles = conf.getInt(OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES, OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT); setStore(loadDB(conf, dir, name, true, Optional.of(Boolean.TRUE), - maxOpenFiles, false, false, true)); + maxOpenFiles, false, false, true, null)); initializeOmTables(CacheType.PARTIAL_CACHE, false); perfMetrics = null; } @@ -428,7 +428,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) setStore(loadDB(conf, metaDir, dbName, false, java.util.Optional.of(Boolean.TRUE), maxOpenFiles, false, false, conf.getBoolean(OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED, - OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED_DEFAULT))); + OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED_DEFAULT), null)); initializeOmTables(CacheType.PARTIAL_CACHE, false); } catch (IOException e) { stop(); @@ -562,7 +562,7 @@ public void start(OzoneConfiguration configuration) throws IOException { int maxOpenFiles = configuration.getInt(OZONE_OM_DB_MAX_OPEN_FILES, OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT); - this.store = loadDB(configuration, metaDir, maxOpenFiles); + this.store = loadDB(configuration, metaDir, maxOpenFiles, getLock()); initializeOmTables(CacheType.FULL_CACHE, true); } @@ -570,9 +570,10 @@ public void start(OzoneConfiguration configuration) throws IOException { snapshotChainManager = new SnapshotChainManager(this); } - public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles) throws IOException { + public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles, + IOzoneManagerLock lock) throws IOException { return loadDB(configuration, metaDir, OM_DB_NAME, false, - java.util.Optional.empty(), maxOpenFiles, true, true, true); + java.util.Optional.empty(), maxOpenFiles, true, true, true, lock); } @SuppressWarnings("checkstyle:parameternumber") @@ -582,7 +583,8 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles, boolean enableCompactionDag, boolean createCheckpointDirs, - boolean enableRocksDBMetrics) + boolean enableRocksDBMetrics, + IOzoneManagerLock lock) throws IOException { RocksDBConfiguration rocksDBConfiguration = configuration.getObject(RocksDBConfiguration.class); @@ -594,6 +596,9 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, .setCreateCheckpointDirs(createCheckpointDirs) .setMaxNumberOfOpenFiles(maxOpenFiles) .setEnableRocksDbMetrics(enableRocksDBMetrics); + if (lock != null) { + dbStoreBuilder = dbStoreBuilder.setLock(lock); + } disableAutoCompaction.ifPresent( dbStoreBuilder::disableDefaultCFAutoCompaction); return addOMTablesAndCodecs(dbStoreBuilder).build(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java index 395b37e859c3..263a389ffbda 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.om.codec.OMDBDefinition; +import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -51,7 +52,8 @@ public void testDBDefinition() throws IOException { final int countOmDefTables = columnFamilyDefinitions.size(); List missingDBDefTables = new ArrayList<>(); - try (DBStore store = OmMetadataManagerImpl.loadDB(configuration, metaDir, -1)) { + try (DBStore store = OmMetadataManagerImpl.loadDB(configuration, metaDir, -1, + new OzoneManagerLock(configuration))) { // Get list of tables from the RocksDB Store final Collection missingOmDBTables = new ArrayList<>(store.getTableNames().values()); missingOmDBTables.remove("default"); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java index 74ae33ae956a..eb6c5a0eaea2 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java @@ -44,6 +44,7 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; +import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.repair.RepairTool; import org.apache.ratis.util.Preconditions; @@ -508,7 +509,9 @@ protected static DBStore getStoreFromPath(String dbPath) throws IOException { "not exist or is not a RocksDB directory.", dbPath)); } // Load RocksDB and tables needed. - return OmMetadataManagerImpl.loadDB(new OzoneConfiguration(), new File(dbPath).getParentFile(), -1); + OzoneConfiguration conf = new OzoneConfiguration(); + return OmMetadataManagerImpl.loadDB(conf, new File(dbPath).getParentFile(), -1, + new OzoneManagerLock(conf)); } /** From d3112559667ed15aebdb265460ad1a110b7fe249 Mon Sep 17 00:00:00 2001 From: saketa Date: Mon, 28 Apr 2025 22:32:35 -0700 Subject: [PATCH 04/18] HDDS-12501. Added Test cases. Addressed comments. --- .../apache/hadoop/ozone/OzoneConfigKeys.java | 11 ++ .../src/main/resources/ozone-default.xml | 10 ++ .../db/managed/ManagedRawSSTFileReader.java | 10 ++ .../rocksdiff/RocksDBCheckpointDiffer.java | 79 +++++------ .../TestRocksDBCheckpointDiffer.java | 124 +++++++++++++++++- .../apache/hadoop/ozone/om/OMConfigKeys.java | 3 - .../ozone/om/snapshot/TestOmSnapshot.java | 50 +++++++ .../om/snapshot/SnapshotDiffManager.java | 10 +- .../om/snapshot/TestSnapshotDiffManager.java | 4 +- 9 files changed, 254 insertions(+), 47 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 99f39f2e0a4b..5a0e09f9391a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -646,6 +646,17 @@ public final class OzoneConfigKeys { OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT = TimeUnit.HOURS.toMillis(1); + public static final String + OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE = + "ozone.om.snapshot.prune.compaction.backup.batch.size"; + + public static final int + OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT = 2000; + + public static final String OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB = + "ozone.om.snapshot.load.native.lib"; + public static final boolean OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT = true; + public static final String OZONE_OM_DELTA_UPDATE_DATA_SIZE_MAX_LIMIT = "ozone.om.delta.update.data.size.max.limit"; public static final String diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 13d18e4acc07..1494eea08136 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4335,6 +4335,16 @@ + + ozone.om.snapshot.prune.compaction.backup.batch.size + 2000 + OZONE, OM + + Prune SST files in Compaction backup directory in batches every + ozone.om.snapshot.compaction.dag.prune.daemon.run.interval. + + + ozone.om.snapshot.compaction.dag.prune.daemon.run.interval 3600s diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java index eadf1afcb879..a84f0a0cd088 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java @@ -32,6 +32,8 @@ */ public class ManagedRawSSTFileReader implements Closeable { + private static boolean isLibraryLoaded = false; + public static boolean tryLoadLibrary() { try { loadLibrary(); @@ -50,6 +52,14 @@ public static boolean loadLibrary() throws NativeLibraryNotLoadedException { return true; } + public static void setIsLibraryLoaded(boolean isLibraryLoaded) { + ManagedRawSSTFileReader.isLibraryLoaded = isLibraryLoaded; + } + + public static boolean isLibraryLoaded() { + return isLibraryLoaded; + } + private final String fileName; // Native address of pointer to the object. private final long nativeHandle; diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index c5de678641c2..145deafdf6d8 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -21,6 +21,10 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX; @@ -177,7 +181,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final StripedLockProvider omLock; private static final String SST_FILE_LOCK = "SST_FILE_LOCK"; private static final int SST_READ_AHEAD_SIZE = 2 * 1024 * 1024; - + private static int pruneSSTFileBatchSize; private ColumnFamilyHandle snapshotInfoTableCFHandle; private static final String DAG_PRUNING_SERVICE_NAME = "CompactionDagPruningService"; private AtomicBoolean suspended; @@ -185,7 +189,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private ColumnFamilyHandle compactionLogTableCFHandle; private ManagedRocksDB activeRocksDB; private ConcurrentMap inflightCompactions; - private boolean isNativeLibsLoaded; private BlockingQueue> pruneQueue = new LinkedBlockingQueue>(); @@ -241,6 +244,9 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + this.pruneSSTFileBatchSize = configuration.getInt( + OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE, + OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT); if (pruneCompactionDagDaemonRunIntervalInMs > 0) { this.scheduler = new Scheduler(DAG_PRUNING_SERVICE_NAME, @@ -258,14 +264,16 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, pruneCompactionDagDaemonRunIntervalInMs, TimeUnit.MILLISECONDS); - try { - isNativeLibsLoaded = ManagedRawSSTFileReader.loadLibrary(); - } catch (NativeLibraryNotLoadedException e) { - LOG.warn("Native Library for raw sst file reading loading failed." + - " Cannot prune OMKeyInfo from SST files. {}", e.getMessage()); - isNativeLibsLoaded = false; + if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT) + && !ManagedRawSSTFileReader.isLibraryLoaded()) { + try { + ManagedRawSSTFileReader.setIsLibraryLoaded(ManagedRawSSTFileReader.loadLibrary()); + } catch (NativeLibraryNotLoadedException e) { + LOG.warn("Native Library for raw sst file reading loading failed." + + " Cannot prune OMKeyInfo from SST files. {}", e.getMessage()); + } } - if (isNativeLibsLoaded) { + if (ManagedRawSSTFileReader.isLibraryLoaded()) { this.scheduler.scheduleWithFixedDelay( this::pruneSstFileValues, pruneCompactionDagDaemonRunIntervalInMs, @@ -562,9 +570,8 @@ public void onCompactionCompleted(RocksDB db, } // Add the compaction log entry to the prune queue // so that the backup input sst files can be pruned. - if (isNativeLibsLoaded) { + if (ManagedRawSSTFileReader.isLibraryLoaded()) { try { - LOG.info("OnCompactionCompleted: {}", compactionLogEntry); pruneQueue.put(Pair.of(key, compactionLogEntry)); } catch (InterruptedException e) { LOG.error("Could not add backup SST files in queue to be pruned. {}", @@ -815,6 +822,15 @@ public void loadAllCompactionLogs() { populateCompactionDAG(compactionLogEntry.getInputFileInfoList(), compactionLogEntry.getOutputFileInfoList(), compactionLogEntry.getDbSequenceNumber()); + // Add the compaction log entry to the prune queue so that the backup input sst files can be pruned. + if (ManagedRawSSTFileReader.isLibraryLoaded()) { + try { + pruneQueue.put(Pair.of(managedRocksIterator.get().key(), compactionLogEntry)); + } catch (InterruptedException e) { + LOG.error("Could not add backup SST files in queue to be pruned. {}", + compactionLogEntry.getInputFileInfoList(), e); + } + } managedRocksIterator.get().next(); } } catch (InvalidProtocolBufferException e) { @@ -889,7 +905,6 @@ public synchronized Optional> getSSTDiffListWithFullPath(DifferSnap String sstFullPath = getSSTFullPath(sst, src.getDbPath(), dest.getDbPath()); Path link = Paths.get(sstFilesDirForSnapDiffJob, sst + SST_FILE_EXTENSION); - // Take a read lock on the SST FILE ReentrantReadWriteLock.ReadLock sstReadLock = getSSTFileLock(sstFullPath).readLock(); sstReadLock.lock(); try { @@ -1193,17 +1208,7 @@ private void addFileInfoToCompactionLogTable( builder.setCompactionReason(compactionReason); } - CompactionLogEntry compactionLogEntry = builder.build(); - byte[] key = addToCompactionLogTable(compactionLogEntry); - // Add the compaction log entry to the prune queue so that the backup input sst files can be pruned. - if (isNativeLibsLoaded) { - try { - pruneQueue.put(Pair.of(key, compactionLogEntry)); - } catch (InterruptedException e) { - LOG.error("Could not add backup SST files in queue to be pruned. {}", - compactionLogEntry.getInputFileInfoList(), e); - } - } + addToCompactionLogTable(builder.build()); } /** @@ -1459,7 +1464,9 @@ public void pruneSstFileValues() { ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { Pair compactionLogTableEntry; - while ((compactionLogTableEntry = pruneQueue.poll()) != null) { + int batchCounter = 0; + while ((compactionLogTableEntry = pruneQueue.poll()) != null + && ++batchCounter <= pruneSSTFileBatchSize) { byte[] key = compactionLogTableEntry.getKey(); CompactionLogEntry compactionLogEntry = compactionLogTableEntry.getValue(); @@ -1472,12 +1479,6 @@ public void pruneSstFileValues() { continue; } String sstFileName = fileInfo.getFileName(); - if (compactionNodeMap.get(sstFileName).isPruned()) { - shouldUpdateTable = true; - fileInfo.setPruned(); - updatedFileInfoList.add(fileInfo); - continue; - } File sstFile = sstBackupDirPath.resolve(sstFileName + ROCKSDB_SST_SUFFIX).toFile(); if (Files.notExists(sstFile.toPath())) { LOG.debug("Skipping pruning SST file {} as it does not exist in backup directory.", sstFile); @@ -1486,7 +1487,10 @@ public void pruneSstFileValues() { } // Write the file.sst => file.sst.tmp - File prunedSSTFile = Files.createFile(sstBackupDirPath.resolve(sstFile.getName() + ".tmp")).toFile(); + Path prunedSSTFilePath = sstBackupDirPath.resolve(sstFile.getName() + ".tmp"); + Files.deleteIfExists(prunedSSTFilePath); + File prunedSSTFile = Files.createFile(prunedSSTFilePath).toFile(); + ReentrantReadWriteLock.WriteLock sstWriteLock = getSSTFileLock(sstFile.getAbsolutePath()).writeLock(); try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( managedOptions, sstFile.getAbsolutePath(), SST_READ_AHEAD_SIZE); @@ -1504,14 +1508,15 @@ public void pruneSstFileValues() { } sstFileWriter.finish(); - // Take a write lock on the SST File sstWriteLock.lock(); - // Move file.sst.tmp to file.sst and replace existing file atomically - Files.move(prunedSSTFile.toPath(), sstFile.toPath(), - StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { + // Move file.sst.tmp to file.sst and replace existing file atomically + Files.move(prunedSSTFile.toPath(), sstFile.toPath(), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } } catch (Exception e) { pruneQueue.put(compactionLogTableEntry); - LOG.error("Could not prune OMKeyInfo from {}. Reprocessing.", sstFileName, e); + LOG.error("Could not prune OMKeyInfo from {}. Reprocessing.", sstFile, e); continue; } finally { Files.deleteIfExists(prunedSSTFile.toPath()); @@ -1521,10 +1526,10 @@ public void pruneSstFileValues() { } // Put pruned flag in compaction DAG. Update CompactionNode in Map. - compactionNodeMap.get(sstFileName).setPruned(); shouldUpdateTable = true; fileInfo.setPruned(); updatedFileInfoList.add(fileInfo); + LOG.debug("Completed pruning OMKeyInfo from {}", sstFile); } if (shouldUpdateTable) { diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index 603ae31a678b..2ff64b54a07a 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -21,9 +21,12 @@ import static java.util.Arrays.asList; import static java.util.concurrent.TimeUnit.MINUTES; import static org.apache.hadoop.hdds.StringUtils.bytes2String; +import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT; import static org.apache.hadoop.util.Time.now; import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG; @@ -46,10 +49,12 @@ import com.google.common.collect.ImmutableSet; import com.google.common.graph.GraphBuilder; import com.google.common.graph.MutableGraph; +import com.google.common.util.concurrent.Striped; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -71,6 +76,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.ReadWriteLock; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -79,16 +85,23 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.SimpleStriped; import org.apache.hadoop.hdds.utils.db.managed.ManagedCheckpoint; import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedFlushOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileIterator; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader; +import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; +import org.apache.hadoop.ozone.lock.StripedLockProvider; import org.apache.hadoop.util.Time; import org.apache.ozone.compaction.log.CompactionFileInfo; import org.apache.ozone.compaction.log.CompactionLogEntry; @@ -99,6 +112,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -184,11 +198,15 @@ public void init() throws RocksDBException { OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS)).thenReturn(0L); + when(config.getInt( + OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE, + OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT)).thenReturn(2000); + rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer(METADATA_DIR_NAME, SST_BACK_UP_DIR_NAME, COMPACTION_LOG_DIR_NAME, ACTIVE_DB_DIR_NAME, - config, null); + config, new StripedLock()); ManagedColumnFamilyOptions cfOpts = new ManagedColumnFamilyOptions(); cfOpts.optimizeUniversalStyleCompaction(); @@ -1891,6 +1909,101 @@ private static Stream casesGetSSTDiffListWithoutDB2() { ); } + + /** + * Test that backup SST files are pruned on loading previous compaction logs. + */ + @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true") + @Test + public void testPruneSSTFileValues() + throws Exception { + ManagedRawSSTFileReader.setIsLibraryLoaded(ManagedRawSSTFileReader.loadLibrary()); + // Create src files in backup directory. + List filesInBackupDir = Arrays.asList("000078", "000073"); + for (String fileName : filesInBackupDir) { + createSSTFileWithKeys(sstBackUpDir.toPath(), fileName, 1, 1); + } + + // Create compacted files in active DB directory. + createSSTFileWithKeys(activeDbDir.toPath(), "000081", 1, 1); + + // Load dummy previous compaction logs. + CompactionLogEntry dummyEntry = new CompactionLogEntry(178, System.currentTimeMillis(), + Arrays.asList(new CompactionFileInfo("000078", + "/volume/bucket1/key-0000001411", + "/volume/bucket2/key-0000099649", + "keyTable"), + new CompactionFileInfo("000073", + "/volume/bucket1/key-0000000730", + "/volume/bucket2/key-0000097010", + "keyTable")), + Collections.singletonList(new CompactionFileInfo("000081", + "/volume/bucket1/key-0000000730", + "/volume/bucket2/key-0000099649", + "keyTable")), + null + ); + rocksDBCheckpointDiffer.addToCompactionLogTable(dummyEntry); + rocksDBCheckpointDiffer.loadAllCompactionLogs(); + + // Run the SST file pruner. + waitForLock(rocksDBCheckpointDiffer, RocksDBCheckpointDiffer::pruneSstFileValues); + + // Verify that files are pruned. + try (ManagedRocksIterator managedRocksIterator = new ManagedRocksIterator( + activeRocksDB.get().newIterator(compactionLogTableCFHandle))) { + managedRocksIterator.get().seekToFirst(); + while (managedRocksIterator.get().isValid()) { + byte[] value = managedRocksIterator.get().value(); + CompactionLogEntry compactionLogEntry = + CompactionLogEntry.getFromProtobuf( + CompactionLogEntryProto.parseFrom(value)); + + compactionLogEntry.getInputFileInfoList().forEach( + f -> { + String fileName = f.getFileName(); + File file = sstBackUpDir.toPath().resolve(fileName + SST_FILE_EXTENSION).toFile(); + if (COLUMN_FAMILIES_TO_TRACK_IN_DAG.contains(f.getColumnFamily()) + && filesInBackupDir.contains(fileName)) { + assertTrue(f.isPruned()); + try (ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( + new ManagedOptions(), file.getAbsolutePath(), 2 * 1024 * 1024); + ManagedRawSSTFileIterator itr = sstFileReader.newIterator( + keyValue -> keyValue.getValue(), null, null)) { + while (itr.hasNext()) { + assertEquals(0, itr.next().length); + } + } + } else { + assertFalse(f.isPruned()); + } + }); + managedRocksIterator.get().next(); + } + } + } + + private void createSSTFileWithKeys(Path dir, String fileName, int nKeys, int nTombstones) + throws Exception { + File file = Files.createFile( + dir.resolve(fileName + SST_FILE_EXTENSION)).toFile(); + try (ManagedEnvOptions envOptions = new ManagedEnvOptions(); + ManagedOptions managedOptions = new ManagedOptions(); + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { + + sstFileWriter.open(file.getAbsolutePath()); + String generatedString = RandomStringUtils.randomAlphabetic(7); + for (int n = 0; n < nKeys; n++) { + sstFileWriter.put(("key" + n + "-" + generatedString).getBytes(StandardCharsets.UTF_8), + ("value" + n + "-" + generatedString).getBytes(StandardCharsets.UTF_8)); + } + for (int n = nKeys; n < nKeys + nTombstones; n++) { + sstFileWriter.delete(("key" + n + "-" + generatedString).getBytes(StandardCharsets.UTF_8)); + } + sstFileWriter.finish(); + } + } + /** * Tests core SST diff list logic. Does not involve DB. * Focuses on testing edge cases in internalGetSSTDiffList(). @@ -2114,4 +2227,13 @@ public void testShouldSkipFile(String description, assertEquals(expectedResult, rocksDBCheckpointDiffer .shouldSkipCompaction(columnFamilyBytes, inputFiles, outputFiles)); } + + private class StripedLock implements StripedLockProvider { + + @Override + public Striped getStripedLock(String key) { + return SimpleStriped.readWriteLock(512, false); + } + + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index b45fc26462f0..baa65a7c2de0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -27,9 +27,6 @@ * Ozone Manager Constants. */ public final class OMConfigKeys { - public static final String OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB = - "ozone.om.snapshot.load.native.lib"; - public static final boolean OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT = true; /** * Never constructed. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index fcc7aaa73459..bcdde1ec17e0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -21,7 +21,9 @@ import static org.apache.commons.lang3.StringUtils.leftPad; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConsts.COMPACTION_LOG_TABLE; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; @@ -90,12 +92,17 @@ import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.DBProfile; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.hdds.utils.db.RocksDatabase; +import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; @@ -137,6 +144,7 @@ import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.apache.ozone.compaction.log.CompactionLogEntry; import org.apache.ozone.rocksdiff.CompactionNode; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Slow; @@ -216,6 +224,9 @@ private void init() throws Exception { conf.setInt(OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion()); conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, KeyManagerImpl.DISABLE_VALUE); + if (!disableNativeDiff) { + conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 1, TimeUnit.SECONDS); + } cluster = MiniOzoneCluster.newBuilder(conf) .build(); @@ -2478,6 +2489,45 @@ public void testSnapshotCompactionDag() throws Exception { assertEquals(200, fetchReportPage(volume1, bucket3, "bucket3-snap1", "bucket3-snap3", null, 0).getDiffList().size()); + + if (!disableNativeDiff) { + // Verify backup SST files are pruned on DB compactions. + Thread.sleep(1000); + + RocksDatabase db = getRdbStore().getDb(); + java.nio.file.Path sstBackUpDir = java.nio.file.Paths.get(getRdbStore() + .getRocksDBCheckpointDiffer().getSSTBackupDir()); + + try (ManagedRocksIterator managedRocksIterator = new ManagedRocksIterator( + db.getManagedRocksDb().get().newIterator(db.getColumnFamily(COMPACTION_LOG_TABLE).getHandle()))) { + managedRocksIterator.get().seekToFirst(); + while (managedRocksIterator.get().isValid()) { + byte[] value = managedRocksIterator.get().value(); + CompactionLogEntry compactionLogEntry = + CompactionLogEntry.getFromProtobuf( + CompactionLogEntryProto.parseFrom(value)); + + compactionLogEntry.getInputFileInfoList().forEach( + f -> { + java.nio.file.Path file = sstBackUpDir.resolve(f.getFileName() + ".sst"); + if (COLUMN_FAMILIES_TO_TRACK_IN_DAG.contains(f.getColumnFamily()) && + java.nio.file.Files.exists(file)) { + assertTrue(f.isPruned()); + try (ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( + new ManagedOptions(), file.toFile().getAbsolutePath(), 2 * 1024 * 1024); + ManagedRawSSTFileIterator itr = sstFileReader.newIterator( + keyValue -> keyValue.getValue(), null, null)) { + while (itr.hasNext()) { + assertEquals(0, itr.next().length); + } + } + } else { + assertFalse(f.isPruned()); + } + }); + } + } + } } @Test diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 285874a86788..a47b97bb29d9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -22,6 +22,8 @@ import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType.DELETE; import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType.MODIFY; import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType.RENAME; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS; @@ -34,8 +36,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_THREAD_POOL_SIZE_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive; @@ -285,9 +285,11 @@ public PersistentMap getSnapDiffJobTable() { } private boolean initNativeLibraryForEfficientDiff(final OzoneConfiguration conf) { - if (conf.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { + if (conf.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT) + && !ManagedRawSSTFileReader.isLibraryLoaded()) { try { - return ManagedRawSSTFileReader.loadLibrary(); + ManagedRawSSTFileReader.setIsLibraryLoaded(ManagedRawSSTFileReader.loadLibrary()); + return ManagedRawSSTFileReader.isLibraryLoaded(); } catch (NativeLibraryNotLoadedException e) { LOG.warn("Native Library for raw sst file reading loading failed." + " Fallback to performing a full diff instead. {}", e.getMessage()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 281f6a7e31fc..68338ac0f844 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ozone.om.snapshot; import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.DEFAULT_COLUMN_FAMILY_NAME; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_JOB_DEFAULT_WAIT_TIME; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_JOB_DEFAULT_WAIT_TIME_DEFAULT; @@ -27,8 +29,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_THREAD_POOL_SIZE_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER; import static org.apache.hadoop.ozone.om.OmSnapshotManager.SNAP_DIFF_JOB_TABLE_NAME; import static org.apache.hadoop.ozone.om.OmSnapshotManager.SNAP_DIFF_REPORT_TABLE_NAME; From 53c6185b42d21866af8aae480f0b3bfd24a5b935 Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 29 Apr 2025 21:46:51 -0700 Subject: [PATCH 05/18] HDDS-12501. Findbugs and pmd fix. --- .../db/managed/ManagedRawSSTFileReader.java | 19 ++- .../rocksdiff/RocksDBCheckpointDiffer.java | 108 ++++++++++-------- .../TestRocksDBCheckpointDiffer.java | 14 +-- .../ozone/om/OmMetadataManagerImpl.java | 3 +- .../om/snapshot/SnapshotDiffManager.java | 6 +- 5 files changed, 78 insertions(+), 72 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java index aac034448f8b..e312134b5fef 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java @@ -49,22 +49,17 @@ public static boolean tryLoadLibrary() { } public static boolean loadLibrary() throws NativeLibraryNotLoadedException { - ManagedRocksObjectUtils.loadRocksDBLibrary(); - if (!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME, Arrays.asList( - ManagedRocksObjectUtils.getRocksDBLibFileName()))) { - throw new NativeLibraryNotLoadedException(ROCKS_TOOLS_NATIVE_LIBRARY_NAME); + if (!isLibraryLoaded) { + ManagedRocksObjectUtils.loadRocksDBLibrary(); + if (!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME, Arrays.asList( + ManagedRocksObjectUtils.getRocksDBLibFileName()))) { + throw new NativeLibraryNotLoadedException(ROCKS_TOOLS_NATIVE_LIBRARY_NAME); + } + isLibraryLoaded = true; } return true; } - public static void setIsLibraryLoaded(boolean isLibraryLoaded) { - ManagedRawSSTFileReader.isLibraryLoaded = isLibraryLoaded; - } - - public static boolean isLibraryLoaded() { - return isLibraryLoaded; - } - public ManagedRawSSTFileReader(final ManagedOptions options, final String fileName, final int readAheadSize) { this.fileName = fileName; this.nativeHandle = this.newRawSSTFileReader(options.getNativeHandle(), fileName, readAheadSize); diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index b146ff849909..6d03347aa4f4 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -181,7 +181,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final StripedLockProvider omLock; private static final String SST_FILE_LOCK = "SST_FILE_LOCK"; private static final int SST_READ_AHEAD_SIZE = 2 * 1024 * 1024; - private static int pruneSSTFileBatchSize; + private int pruneSSTFileBatchSize; private ColumnFamilyHandle snapshotInfoTableCFHandle; private static final String DAG_PRUNING_SERVICE_NAME = "CompactionDagPruningService"; private AtomicBoolean suspended; @@ -189,6 +189,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private ColumnFamilyHandle compactionLogTableCFHandle; private ManagedRocksDB activeRocksDB; private ConcurrentMap inflightCompactions; + private boolean isNativeLibsLoaded = false; private BlockingQueue> pruneQueue = new LinkedBlockingQueue>(); @@ -293,16 +294,15 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, pruneCompactionDagDaemonRunIntervalInMs, TimeUnit.MILLISECONDS); - if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT) - && !ManagedRawSSTFileReader.isLibraryLoaded()) { + if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { try { - ManagedRawSSTFileReader.setIsLibraryLoaded(ManagedRawSSTFileReader.loadLibrary()); + isNativeLibsLoaded = ManagedRawSSTFileReader.loadLibrary(); } catch (NativeLibraryNotLoadedException e) { LOG.warn("Native Library for raw sst file reading loading failed." + " Cannot prune OMKeyInfo from SST files. {}", e.getMessage()); } } - if (ManagedRawSSTFileReader.isLibraryLoaded()) { + if (isNativeLibsLoaded) { this.scheduler.scheduleWithFixedDelay( this::pruneSstFileValues, pruneCompactionDagDaemonRunIntervalInMs, @@ -570,7 +570,7 @@ public void onCompactionCompleted(RocksDB db, } // Add the compaction log entry to the prune queue // so that the backup input sst files can be pruned. - if (ManagedRawSSTFileReader.isLibraryLoaded()) { + if (isNativeLibsLoaded) { try { pruneQueue.put(Pair.of(key, compactionLogEntry)); } catch (InterruptedException e) { @@ -823,7 +823,7 @@ public void loadAllCompactionLogs() { compactionLogEntry.getOutputFileInfoList(), compactionLogEntry.getDbSequenceNumber()); // Add the compaction log entry to the prune queue so that the backup input sst files can be pruned. - if (ManagedRawSSTFileReader.isLibraryLoaded()) { + if (isNativeLibsLoaded) { try { pruneQueue.put(Pair.of(managedRocksIterator.get().key(), compactionLogEntry)); } catch (InterruptedException e) { @@ -1458,19 +1458,17 @@ public void pruneSstFileValues() { if (!shouldRun()) { return; } - Path sstBackupDirPath = Paths.get(sstBackupDir); - try (ManagedEnvOptions envOptions = new ManagedEnvOptions(); - ManagedOptions managedOptions = new ManagedOptions(); - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { + Path sstBackupDirPath = Paths.get(sstBackupDir); + try (ManagedOptions managedOptions = new ManagedOptions(); + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(new ManagedEnvOptions(), managedOptions)) { Pair compactionLogTableEntry; int batchCounter = 0; - while ((compactionLogTableEntry = pruneQueue.poll()) != null - && ++batchCounter <= pruneSSTFileBatchSize) { - + while ((compactionLogTableEntry = pruneQueue.poll()) != null && ++batchCounter <= pruneSSTFileBatchSize) { byte[] key = compactionLogTableEntry.getKey(); CompactionLogEntry compactionLogEntry = compactionLogTableEntry.getValue(); boolean shouldUpdateTable = false; + boolean shouldReprocess = false; List fileInfoList = compactionLogEntry.getInputFileInfoList(); List updatedFileInfoList = new ArrayList<>(); for (CompactionFileInfo fileInfo : fileInfoList) { @@ -1479,25 +1477,22 @@ public void pruneSstFileValues() { continue; } String sstFileName = fileInfo.getFileName(); - File sstFile = sstBackupDirPath.resolve(sstFileName + ROCKSDB_SST_SUFFIX).toFile(); - if (Files.notExists(sstFile.toPath())) { - LOG.debug("Skipping pruning SST file {} as it does not exist in backup directory.", sstFile); + Path sstFilePath = sstBackupDirPath.resolve(sstFileName + ROCKSDB_SST_SUFFIX); + if (Files.notExists(sstFilePath)) { + LOG.debug("Skipping pruning SST file {} as it does not exist in backup directory.", sstFilePath); updatedFileInfoList.add(fileInfo); continue; } // Write the file.sst => file.sst.tmp - Path prunedSSTFilePath = sstBackupDirPath.resolve(sstFile.getName() + ".tmp"); - Files.deleteIfExists(prunedSSTFilePath); - File prunedSSTFile = Files.createFile(prunedSSTFilePath).toFile(); - - ReentrantReadWriteLock.WriteLock sstWriteLock = getSSTFileLock(sstFile.getAbsolutePath()).writeLock(); + Path prunedSSTFilePath = sstBackupDirPath.resolve(sstFilePath.toFile().getName() + ".tmp"); + ReentrantReadWriteLock.WriteLock sstWriteLock = getSSTFileLock(sstFilePath.toString()).writeLock(); try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( - managedOptions, sstFile.getAbsolutePath(), SST_READ_AHEAD_SIZE); + managedOptions, sstFilePath.toFile().getAbsolutePath(), SST_READ_AHEAD_SIZE); ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null)) { - - sstFileWriter.open(prunedSSTFile.getAbsolutePath()); + Files.deleteIfExists(prunedSSTFilePath); + sstFileWriter.open(prunedSSTFilePath.toFile().getAbsolutePath()); while (itr.hasNext()) { Pair keyValue = itr.next(); if (keyValue.getValue() == 0) { @@ -1506,49 +1501,61 @@ public void pruneSstFileValues() { sstFileWriter.put(keyValue.getKey(), new byte[0]); } } + } catch (Exception e) { + shouldReprocess = true; + try { + Files.deleteIfExists(prunedSSTFilePath); + } catch (IOException ex) { } + continue; + } finally { sstFileWriter.finish(); + } - sstWriteLock.lock(); - try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { - // Move file.sst.tmp to file.sst and replace existing file atomically - Files.move(prunedSSTFile.toPath(), sstFile.toPath(), - StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } + // Move file.sst.tmp to file.sst and replace existing file atomically + sstWriteLock.lock(); + try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { + Files.move(prunedSSTFilePath, sstFilePath, + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (Exception e) { - pruneQueue.put(compactionLogTableEntry); - LOG.error("Could not prune OMKeyInfo from {}. Reprocessing.", sstFile, e); + shouldReprocess = true; continue; } finally { - Files.deleteIfExists(prunedSSTFile.toPath()); - if (sstWriteLock.isHeldByCurrentThread()) { - sstWriteLock.unlock(); - } + sstWriteLock.unlock(); + try { + Files.deleteIfExists(prunedSSTFilePath); + } catch (IOException e) { } } - // Put pruned flag in compaction DAG. Update CompactionNode in Map. shouldUpdateTable = true; fileInfo.setPruned(); updatedFileInfoList.add(fileInfo); - LOG.debug("Completed pruning OMKeyInfo from {}", sstFile); + LOG.debug("Completed pruning OMKeyInfo from {}", sstFilePath); } if (shouldUpdateTable) { // Update Compaction Log table. Track keys that need updating. - CompactionLogEntry.Builder builder = new CompactionLogEntry.Builder( - compactionLogEntry.getDbSequenceNumber(), - compactionLogEntry.getCompactionTime(), - updatedFileInfoList, - compactionLogEntry.getOutputFileInfoList()); + CompactionLogEntry.Builder builder = new CompactionLogEntry.Builder(compactionLogEntry.getDbSequenceNumber(), + compactionLogEntry.getCompactionTime(), updatedFileInfoList, compactionLogEntry.getOutputFileInfoList()); String compactionReason = compactionLogEntry.getCompactionReason(); if (compactionReason != null) { builder.setCompactionReason(compactionReason); } + synchronized (this) { + try { + activeRocksDB.get().put(compactionLogTableCFHandle, key, builder.build().getProtobuf().toByteArray()); + } catch (RocksDBException e) { + shouldReprocess = true; + } + } + } + + if (shouldReprocess) { try { - activeRocksDB.get().put(compactionLogTableCFHandle, key, builder.build().getProtobuf().toByteArray()); - } catch (RocksDBException e) { + LOG.error("Could not prune source files or update the compaction log table. Reprocessing entry: {}", + compactionLogEntry); pruneQueue.put(compactionLogTableEntry); - LOG.error("Could not update pruned flag in compaction log table. Reprocessing entry: {}", - compactionLogEntry, e); + } catch (Exception e) { + LOG.error("Could not reprocess entry: {}", compactionLogEntry, e); } } } @@ -1581,6 +1588,11 @@ public void suspend() { suspended.set(true); } + @VisibleForTesting + void setIsNativeLibsLoaded(boolean isNativeLibsLoaded) { + this.isNativeLibsLoaded = isNativeLibsLoaded; + } + /** * Holder for RocksDBCheckpointDiffer instance. * This is to protect from creating more than one instance of diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index 4ce1e17960bd..968f7376c4fe 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -1915,15 +1915,16 @@ private static Stream casesGetSSTDiffListWithoutDB2() { @Test public void testPruneSSTFileValues() throws Exception { - ManagedRawSSTFileReader.setIsLibraryLoaded(ManagedRawSSTFileReader.loadLibrary()); + rocksDBCheckpointDiffer.setIsNativeLibsLoaded(ManagedRawSSTFileReader.loadLibrary()); // Create src files in backup directory. - List filesInBackupDir = Arrays.asList("000078", "000073"); + //List filesInBackupDir = Arrays.asList("000078", "000073"); + List filesInBackupDir = Arrays.asList("000078.sst", "000078.sst.tmp", "000073.sst"); for (String fileName : filesInBackupDir) { createSSTFileWithKeys(sstBackUpDir.toPath(), fileName, 1, 1); } // Create compacted files in active DB directory. - createSSTFileWithKeys(activeDbDir.toPath(), "000081", 1, 1); + createSSTFileWithKeys(activeDbDir.toPath(), "000081.sst", 1, 1); // Load dummy previous compaction logs. CompactionLogEntry dummyEntry = new CompactionLogEntry(178, System.currentTimeMillis(), @@ -1962,7 +1963,7 @@ public void testPruneSSTFileValues() String fileName = f.getFileName(); File file = sstBackUpDir.toPath().resolve(fileName + SST_FILE_EXTENSION).toFile(); if (COLUMN_FAMILIES_TO_TRACK_IN_DAG.contains(f.getColumnFamily()) - && filesInBackupDir.contains(fileName)) { + && filesInBackupDir.contains(fileName + SST_FILE_EXTENSION)) { assertTrue(f.isPruned()); try (ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( new ManagedOptions(), file.getAbsolutePath(), 2 * 1024 * 1024); @@ -1983,8 +1984,7 @@ public void testPruneSSTFileValues() private void createSSTFileWithKeys(Path dir, String fileName, int nKeys, int nTombstones) throws Exception { - File file = Files.createFile( - dir.resolve(fileName + SST_FILE_EXTENSION)).toFile(); + File file = Files.createFile(dir.resolve(fileName)).toFile(); try (ManagedEnvOptions envOptions = new ManagedEnvOptions(); ManagedOptions managedOptions = new ManagedOptions(); ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { @@ -2226,7 +2226,7 @@ public void testShouldSkipFile(String description, .shouldSkipCompaction(columnFamilyBytes, inputFiles, outputFiles)); } - private class StripedLock implements StripedLockProvider { + static class StripedLock implements StripedLockProvider { @Override public Striped getStripedLock(String key) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 5c8dab3e1eed..818bad249ddb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -542,7 +542,8 @@ public void start(OzoneConfiguration configuration) throws IOException { snapshotChainManager = new SnapshotChainManager(this); } - public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles) throws IOException { + public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles, IOzoneManagerLock lock) + throws IOException { return newDBStoreBuilder(configuration, null, metaDir) .setOpenReadOnly(false) .setEnableCompactionDag(true) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 96f8cbbd4d57..f42e80f56fb5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -285,11 +285,9 @@ public PersistentMap getSnapDiffJobTable() { } private boolean initNativeLibraryForEfficientDiff(final OzoneConfiguration conf) { - if (conf.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT) - && !ManagedRawSSTFileReader.isLibraryLoaded()) { + if (conf.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { try { - ManagedRawSSTFileReader.setIsLibraryLoaded(ManagedRawSSTFileReader.loadLibrary()); - return ManagedRawSSTFileReader.isLibraryLoaded(); + return ManagedRawSSTFileReader.loadLibrary(); } catch (NativeLibraryNotLoadedException e) { LOG.warn("Native Library for raw sst file reading loading failed." + " Fallback to performing a full diff instead. {}", e.getMessage()); From 7cfbb1ddab0d0ea66e8c6fde7f828f836f2f8032 Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 30 Apr 2025 11:28:10 -0700 Subject: [PATCH 06/18] HDDS-12501. More findbugs fixes. Fixed exception handling. --- .../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 16 +++++++++------- .../rocksdiff/TestRocksDBCheckpointDiffer.java | 1 - 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 6d03347aa4f4..b1856c84ef19 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -1501,14 +1501,18 @@ public void pruneSstFileValues() { sstFileWriter.put(keyValue.getKey(), new byte[0]); } } - } catch (Exception e) { + } catch (IOException | RocksDBException e) { shouldReprocess = true; try { Files.deleteIfExists(prunedSSTFilePath); } catch (IOException ex) { } continue; } finally { - sstFileWriter.finish(); + try { + sstFileWriter.finish(); + } catch (RocksDBException e) { + throw new RuntimeException("Failed to finish writing to " + prunedSSTFilePath, e); + } } // Move file.sst.tmp to file.sst and replace existing file atomically @@ -1516,7 +1520,7 @@ public void pruneSstFileValues() { try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { Files.move(prunedSSTFilePath, sstFilePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception e) { + } catch (IOException | InterruptedException e) { shouldReprocess = true; continue; } finally { @@ -1554,13 +1558,11 @@ public void pruneSstFileValues() { LOG.error("Could not prune source files or update the compaction log table. Reprocessing entry: {}", compactionLogEntry); pruneQueue.put(compactionLogTableEntry); - } catch (Exception e) { + } catch (InterruptedException e) { LOG.error("Could not reprocess entry: {}", compactionLogEntry, e); } } } - } catch (Exception e) { - LOG.error("Could not prune OMKeyInfo from SST files.", e); } } @@ -1589,7 +1591,7 @@ public void suspend() { } @VisibleForTesting - void setIsNativeLibsLoaded(boolean isNativeLibsLoaded) { + synchronized void setIsNativeLibsLoaded(boolean isNativeLibsLoaded) { this.isNativeLibsLoaded = isNativeLibsLoaded; } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index 968f7376c4fe..3aa7c1d93dea 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -1917,7 +1917,6 @@ public void testPruneSSTFileValues() throws Exception { rocksDBCheckpointDiffer.setIsNativeLibsLoaded(ManagedRawSSTFileReader.loadLibrary()); // Create src files in backup directory. - //List filesInBackupDir = Arrays.asList("000078", "000073"); List filesInBackupDir = Arrays.asList("000078.sst", "000078.sst.tmp", "000073.sst"); for (String fileName : filesInBackupDir) { createSSTFileWithKeys(sstBackUpDir.toPath(), fileName, 1, 1); From 17d24c1822ab4072603fe5a3b6b9d8bf2582c5c4 Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 30 Apr 2025 19:42:09 -0700 Subject: [PATCH 07/18] HDDS-12501. Removed locks and native library loader optimization. --- .../ozone/lock/StripedLockProvider.java | 28 ------ .../hadoop/hdds/utils/db/DBStoreBuilder.java | 9 +- .../apache/hadoop/hdds/utils/db/RDBStore.java | 7 +- .../hadoop/hdds/utils/db/TestRDBStore.java | 2 +- .../db/managed/ManagedRawSSTFileReader.java | 12 +-- .../ozone/rocksdiff/CompactionNode.java | 19 +--- .../rocksdiff/RocksDBCheckpointDiffer.java | 90 ++++++------------- .../log/TestCompactionFileInfo.java | 29 +++++- .../TestRocksDBCheckpointDiffer.java | 35 +++----- .../ozone/om/lock/IOzoneManagerLock.java | 3 +- .../hadoop/ozone/om/lock/OmReadOnlyLock.java | 8 -- .../ozone/om/lock/OzoneManagerLock.java | 8 +- .../ozone/om/snapshot/TestOmSnapshot.java | 29 +++--- .../ozone/om/OmMetadataManagerImpl.java | 6 +- .../hadoop/ozone/om/TestOMDBDefinition.java | 4 +- .../hadoop/ozone/repair/om/FSORepairTool.java | 5 +- 16 files changed, 91 insertions(+), 203 deletions(-) delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/StripedLockProvider.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/StripedLockProvider.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/StripedLockProvider.java deleted file mode 100644 index a05fa4d2b485..000000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/StripedLockProvider.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.lock; - -import com.google.common.util.concurrent.Striped; -import java.util.concurrent.locks.ReadWriteLock; - -/** - * Interface for locks. - */ -public interface StripedLockProvider { - Striped getStripedLock(String key); -} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index 6ecb2d1fbd6b..62d896d133c4 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -51,7 +51,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedStatistics; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions; -import org.apache.hadoop.ozone.lock.StripedLockProvider; import org.eclipse.jetty.util.StringUtil; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.InfoLogLevel; @@ -105,7 +104,6 @@ public final class DBStoreBuilder { // number in request to avoid increase in heap memory. private long maxDbUpdatesSizeThreshold; private Integer maxNumberOfOpenFiles = null; - private StripedLockProvider lock = null; /** * Create DBStoreBuilder from a generic DBDefinition. @@ -228,7 +226,7 @@ public RDBStore build() throws IOException { return new RDBStore(dbFile, rocksDBOption, statistics, writeOptions, tableConfigs, openReadOnly, dbJmxBeanNameName, enableCompactionDag, maxDbUpdatesSizeThreshold, createCheckpointDirs, configuration, - enableRocksDbMetrics, lock); + enableRocksDbMetrics); } finally { tableConfigs.forEach(TableConfig::close); } @@ -291,11 +289,6 @@ public DBStoreBuilder setEnableRocksDbMetrics(boolean enableRocksDbMetrics) { return this; } - public DBStoreBuilder setLock(StripedLockProvider lock) { - this.lock = lock; - return this; - } - /** * Set the {@link ManagedDBOptions} and default * {@link ManagedColumnFamilyOptions} based on {@code prof}. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 39db07ad8673..b0096730d01e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedStatistics; import org.apache.hadoop.hdds.utils.db.managed.ManagedTransactionLogIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions; -import org.apache.hadoop.ozone.lock.StripedLockProvider; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.RocksDBCheckpointDifferHolder; import org.rocksdb.RocksDBException; @@ -83,8 +82,7 @@ public class RDBStore implements DBStore { long maxDbUpdatesSizeThreshold, boolean createCheckpointDirs, ConfigurationSource configuration, - boolean enableRocksDBMetrics, - StripedLockProvider lock) + boolean enableRocksDBMetrics) throws IOException { Preconditions.checkNotNull(dbFile, "DB file location cannot be null"); @@ -103,8 +101,7 @@ public class RDBStore implements DBStore { DB_COMPACTION_SST_BACKUP_DIR, DB_COMPACTION_LOG_DIR, dbLocation.toString(), - configuration, - lock); + configuration); rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions); } else { rocksDBCheckpointDiffer = null; diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java index 30153b750856..567c2a13c011 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java @@ -89,7 +89,7 @@ public static RDBStore newRDBStore(File dbFile, ManagedDBOptions options, throws IOException { return new RDBStore(dbFile, options, null, new ManagedWriteOptions(), families, false, null, false, - maxDbUpdatesSizeThreshold, true, null, true, null); + maxDbUpdatesSizeThreshold, true, null, true); } @BeforeEach diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java index e312134b5fef..461d9ddc8ab9 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java @@ -32,7 +32,6 @@ */ public class ManagedRawSSTFileReader implements Closeable { - private static boolean isLibraryLoaded = false; private static final Logger LOG = LoggerFactory.getLogger(ManagedRawSSTFileReader.class); private final String fileName; @@ -49,13 +48,10 @@ public static boolean tryLoadLibrary() { } public static boolean loadLibrary() throws NativeLibraryNotLoadedException { - if (!isLibraryLoaded) { - ManagedRocksObjectUtils.loadRocksDBLibrary(); - if (!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME, Arrays.asList( - ManagedRocksObjectUtils.getRocksDBLibFileName()))) { - throw new NativeLibraryNotLoadedException(ROCKS_TOOLS_NATIVE_LIBRARY_NAME); - } - isLibraryLoaded = true; + ManagedRocksObjectUtils.loadRocksDBLibrary(); + if (!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME, Arrays.asList( + ManagedRocksObjectUtils.getRocksDBLibFileName()))) { + throw new NativeLibraryNotLoadedException(ROCKS_TOOLS_NATIVE_LIBRARY_NAME); } return true; } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java index b7c15a899dc3..6054a55e608d 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java @@ -31,7 +31,6 @@ public class CompactionNode { private final String startKey; private final String endKey; private final String columnFamily; - private boolean pruned; /** * CompactionNode constructor. @@ -39,15 +38,8 @@ public class CompactionNode { * @param numKeys Number of keys in the SST * @param seqNum Snapshot generation (sequence number) */ - public CompactionNode(String file, long numKeys, long seqNum, String startKey, String endKey, String columnFamily) { - this(file, numKeys, seqNum, startKey, endKey, columnFamily, false); - } - - public CompactionNode(String file, long numKeys, long seqNum, - String startKey, String endKey, String columnFamily, - boolean pruned) { fileName = file; totalNumberOfKeys = numKeys; snapshotGeneration = seqNum; @@ -55,12 +47,11 @@ public CompactionNode(String file, long numKeys, long seqNum, this.startKey = startKey; this.endKey = endKey; this.columnFamily = columnFamily; - this.pruned = pruned; } public CompactionNode(CompactionFileInfo compactionFileInfo) { this(compactionFileInfo.getFileName(), -1, -1, compactionFileInfo.getStartKey(), - compactionFileInfo.getEndKey(), compactionFileInfo.getColumnFamily(), compactionFileInfo.isPruned()); + compactionFileInfo.getEndKey(), compactionFileInfo.getColumnFamily()); } @Override @@ -96,10 +87,6 @@ public String getColumnFamily() { return columnFamily; } - public boolean isPruned() { - return pruned; - } - public void setCumulativeKeysReverseTraversal( long cumulativeKeysReverseTraversal) { this.cumulativeKeysReverseTraversal = cumulativeKeysReverseTraversal; @@ -108,8 +95,4 @@ public void setCumulativeKeysReverseTraversal( public void addCumulativeKeysReverseTraversal(long diff) { this.cumulativeKeysReverseTraversal += diff; } - - public void setPruned() { - this.pruned = true; - } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index b1856c84ef19..ed00c6d285d2 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -56,13 +56,11 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.collections.CollectionUtils; @@ -84,7 +82,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; -import org.apache.hadoop.ozone.lock.StripedLockProvider; import org.apache.ozone.compaction.log.CompactionFileInfo; import org.apache.ozone.compaction.log.CompactionLogEntry; import org.apache.ozone.graph.PrintableGraph; @@ -178,8 +175,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final long maxAllowedTimeInDag; private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock(); - private final StripedLockProvider omLock; - private static final String SST_FILE_LOCK = "SST_FILE_LOCK"; private static final int SST_READ_AHEAD_SIZE = 2 * 1024 * 1024; private int pruneSSTFileBatchSize; private ColumnFamilyHandle snapshotInfoTableCFHandle; @@ -190,8 +185,8 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private ManagedRocksDB activeRocksDB; private ConcurrentMap inflightCompactions; private boolean isNativeLibsLoaded = false; - private BlockingQueue> pruneQueue = - new LinkedBlockingQueue>(); + private ConcurrentLinkedQueue> pruneQueue = + new ConcurrentLinkedQueue>(); /** * For snapshot diff calculation we only need to track following column @@ -247,8 +242,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, String sstBackupDirName, String compactionLogDirName, String activeDBLocationName, - ConfigurationSource configuration, - StripedLockProvider omLock) { + ConfigurationSource configuration) { Preconditions.checkNotNull(metadataDirName); Preconditions.checkNotNull(sstBackupDirName); Preconditions.checkNotNull(compactionLogDirName); @@ -259,7 +253,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, createCompactionLogDir(metadataDirName, compactionLogDirName); this.sstBackupDir = Paths.get(metadataDirName, sstBackupDirName) + "/"; createSstBackUpDir(); - this.omLock = omLock; // Active DB location is used in getSSTFileSummary this.activeDBLocationStr = activeDBLocationName + "/"; @@ -274,9 +267,18 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + this.pruneSSTFileBatchSize = configuration.getInt( OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE, OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT); + if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { + try { + this.isNativeLibsLoaded = ManagedRawSSTFileReader.loadLibrary(); + } catch (NativeLibraryNotLoadedException e) { + LOG.warn("Native Library for raw sst file reading loading failed." + + " Cannot prune OMKeyInfo from SST files. {}", e.getMessage()); + } + } if (pruneCompactionDagDaemonRunIntervalInMs > 0) { this.scheduler = new Scheduler(DAG_PRUNING_SERVICE_NAME, @@ -294,14 +296,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, pruneCompactionDagDaemonRunIntervalInMs, TimeUnit.MILLISECONDS); - if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { - try { - isNativeLibsLoaded = ManagedRawSSTFileReader.loadLibrary(); - } catch (NativeLibraryNotLoadedException e) { - LOG.warn("Native Library for raw sst file reading loading failed." + - " Cannot prune OMKeyInfo from SST files. {}", e.getMessage()); - } - } if (isNativeLibsLoaded) { this.scheduler.scheduleWithFixedDelay( this::pruneSstFileValues, @@ -571,12 +565,7 @@ public void onCompactionCompleted(RocksDB db, // Add the compaction log entry to the prune queue // so that the backup input sst files can be pruned. if (isNativeLibsLoaded) { - try { - pruneQueue.put(Pair.of(key, compactionLogEntry)); - } catch (InterruptedException e) { - LOG.error("Could not add backup SST files in queue to be pruned. {}", - compactionLogEntry.getInputFileInfoList(), e); - } + pruneQueue.offer(Pair.of(key, compactionLogEntry)); } } }; @@ -824,12 +813,7 @@ public void loadAllCompactionLogs() { compactionLogEntry.getDbSequenceNumber()); // Add the compaction log entry to the prune queue so that the backup input sst files can be pruned. if (isNativeLibsLoaded) { - try { - pruneQueue.put(Pair.of(managedRocksIterator.get().key(), compactionLogEntry)); - } catch (InterruptedException e) { - LOG.error("Could not add backup SST files in queue to be pruned. {}", - compactionLogEntry.getInputFileInfoList(), e); - } + pruneQueue.offer(Pair.of(managedRocksIterator.get().key(), compactionLogEntry)); } managedRocksIterator.get().next(); } @@ -906,15 +890,9 @@ public synchronized Optional> getSSTDiffListWithFullPath(DifferSnap String sstFullPath = getSSTFullPath(sst, src.getDbPath(), dest.getDbPath()); Path link = Paths.get(sstFilesDirForSnapDiffJob, sst + SST_FILE_EXTENSION); - ReentrantReadWriteLock.ReadLock sstReadLock = getSSTFileLock(sstFullPath).readLock(); - sstReadLock.lock(); - try { - Path srcFile = Paths.get(sstFullPath); - createLink(link, srcFile); - return link.toString(); - } finally { - sstReadLock.unlock(); - } + Path srcFile = Paths.get(sstFullPath); + createLink(link, srcFile); + return link.toString(); }) .collect(Collectors.toList())); } @@ -1461,7 +1439,8 @@ public void pruneSstFileValues() { Path sstBackupDirPath = Paths.get(sstBackupDir); try (ManagedOptions managedOptions = new ManagedOptions(); - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(new ManagedEnvOptions(), managedOptions)) { + ManagedEnvOptions envOptions = new ManagedEnvOptions(); + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { Pair compactionLogTableEntry; int batchCounter = 0; while ((compactionLogTableEntry = pruneQueue.poll()) != null && ++batchCounter <= pruneSSTFileBatchSize) { @@ -1486,7 +1465,6 @@ public void pruneSstFileValues() { // Write the file.sst => file.sst.tmp Path prunedSSTFilePath = sstBackupDirPath.resolve(sstFilePath.toFile().getName() + ".tmp"); - ReentrantReadWriteLock.WriteLock sstWriteLock = getSSTFileLock(sstFilePath.toString()).writeLock(); try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( managedOptions, sstFilePath.toFile().getAbsolutePath(), SST_READ_AHEAD_SIZE); ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( @@ -1516,7 +1494,6 @@ public void pruneSstFileValues() { } // Move file.sst.tmp to file.sst and replace existing file atomically - sstWriteLock.lock(); try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { Files.move(prunedSSTFilePath, sstFilePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); @@ -1524,12 +1501,10 @@ public void pruneSstFileValues() { shouldReprocess = true; continue; } finally { - sstWriteLock.unlock(); try { Files.deleteIfExists(prunedSSTFilePath); } catch (IOException e) { } } - shouldUpdateTable = true; fileInfo.setPruned(); updatedFileInfoList.add(fileInfo); @@ -1554,13 +1529,9 @@ public void pruneSstFileValues() { } if (shouldReprocess) { - try { - LOG.error("Could not prune source files or update the compaction log table. Reprocessing entry: {}", - compactionLogEntry); - pruneQueue.put(compactionLogTableEntry); - } catch (InterruptedException e) { - LOG.error("Could not reprocess entry: {}", compactionLogEntry, e); - } + LOG.error("Could not prune source files or update the compaction log table. Reprocessing entry: {}", + compactionLogEntry); + pruneQueue.offer(compactionLogTableEntry); } } } @@ -1590,11 +1561,6 @@ public void suspend() { suspended.set(true); } - @VisibleForTesting - synchronized void setIsNativeLibsLoaded(boolean isNativeLibsLoaded) { - this.isNativeLibsLoaded = isNativeLibsLoaded; - } - /** * Holder for RocksDBCheckpointDiffer instance. * This is to protect from creating more than one instance of @@ -1610,16 +1576,14 @@ public static RocksDBCheckpointDiffer getInstance( String sstBackupDirName, String compactionLogDirName, String activeDBLocationName, - ConfigurationSource configuration, - StripedLockProvider omLock + ConfigurationSource configuration ) { return INSTANCE_MAP.computeIfAbsent(metadataDirName, (key) -> new RocksDBCheckpointDiffer(metadataDirName, sstBackupDirName, compactionLogDirName, activeDBLocationName, - configuration, - omLock)); + configuration)); } /** @@ -1637,10 +1601,6 @@ public BootstrapStateHandler.Lock getBootstrapStateLock() { return lock; } - public ReentrantReadWriteLock getSSTFileLock(String key) { - return (ReentrantReadWriteLock) omLock.getStripedLock(SST_FILE_LOCK).get(key); - } - public void pngPrintMutableGraph(String filePath, GraphType graphType) throws IOException { Objects.requireNonNull(filePath, "Image file path is required."); diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/compaction/log/TestCompactionFileInfo.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/compaction/log/TestCompactionFileInfo.java index 88311173e3ed..5178bfda7efe 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/compaction/log/TestCompactionFileInfo.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/compaction/log/TestCompactionFileInfo.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.stream.Stream; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionFileInfoProto; @@ -41,6 +42,12 @@ private static Stream compactionFileInfoValidScenarios() { "endRange", "columnFamily" ), + Arguments.of("Only fileName is present.", + "fileName", + null, + null, + null + ), Arguments.of("Only fileName is present.", "fileName", null, @@ -58,10 +65,15 @@ public void testCompactionFileInfoValidScenario(String description, String endRange, String columnFamily) { - CompactionFileInfo compactionFileInfo = - new CompactionFileInfo.Builder(fileName).setStartRange(startRange) - .setEndRange(endRange).setColumnFamily(columnFamily).build(); + CompactionFileInfo.Builder builder = new CompactionFileInfo.Builder(fileName).setStartRange(startRange) + .setEndRange(endRange).setColumnFamily(columnFamily); + CompactionFileInfo compactionFileInfo = builder.build(); assertNotNull(compactionFileInfo); + CompactionFileInfo prunedCompactionFileInfo = builder.setPruned().build(); + assertFalse(compactionFileInfo.isPruned()); + compactionFileInfo.setPruned(); + assertTrue(compactionFileInfo.isPruned()); + assertTrue(prunedCompactionFileInfo.isPruned()); } private static Stream compactionFileInfoInvalidScenarios() { @@ -215,5 +227,16 @@ public void testFromProtobuf(String description, assertEquals(startRange, compactionFileInfo.getStartKey()); assertEquals(endRange, compactionFileInfo.getEndKey()); assertEquals(columnFamily, compactionFileInfo.getColumnFamily()); + assertFalse(compactionFileInfo.isPruned()); + + CompactionFileInfoProto unPrunedProtobuf = builder.setPruned(false).build(); + CompactionFileInfo unPrunedCompactionFileInfo = + CompactionFileInfo.getFromProtobuf(unPrunedProtobuf); + assertFalse(unPrunedCompactionFileInfo.isPruned()); + + CompactionFileInfoProto prunedProtobuf = builder.setPruned(true).build(); + CompactionFileInfo prunedCompactionFileInfo = + CompactionFileInfo.getFromProtobuf(prunedProtobuf); + assertTrue(prunedCompactionFileInfo.isPruned()); } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index 3aa7c1d93dea..162a77849897 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -25,6 +25,8 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT; @@ -49,7 +51,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.graph.GraphBuilder; import com.google.common.graph.MutableGraph; -import com.google.common.util.concurrent.Striped; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -76,7 +77,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.locks.ReadWriteLock; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -87,7 +87,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.hdds.utils.SimpleStriped; import org.apache.hadoop.hdds.utils.db.managed.ManagedCheckpoint; import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; @@ -101,7 +100,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; -import org.apache.hadoop.ozone.lock.StripedLockProvider; import org.apache.hadoop.util.Time; import org.apache.ozone.compaction.log.CompactionFileInfo; import org.apache.ozone.compaction.log.CompactionLogEntry; @@ -359,11 +357,15 @@ public void init() throws RocksDBException { OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE, OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT)).thenReturn(2000); + when(config.getBoolean( + OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, + OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)).thenReturn(true); + rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer(METADATA_DIR_NAME, SST_BACK_UP_DIR_NAME, COMPACTION_LOG_DIR_NAME, ACTIVE_DB_DIR_NAME, - config, new StripedLock()); + config); ManagedColumnFamilyOptions cfOpts = new ManagedColumnFamilyOptions(); cfOpts.optimizeUniversalStyleCompaction(); @@ -1915,7 +1917,6 @@ private static Stream casesGetSSTDiffListWithoutDB2() { @Test public void testPruneSSTFileValues() throws Exception { - rocksDBCheckpointDiffer.setIsNativeLibsLoaded(ManagedRawSSTFileReader.loadLibrary()); // Create src files in backup directory. List filesInBackupDir = Arrays.asList("000078.sst", "000078.sst.tmp", "000073.sst"); for (String fileName : filesInBackupDir) { @@ -1953,10 +1954,8 @@ public void testPruneSSTFileValues() managedRocksIterator.get().seekToFirst(); while (managedRocksIterator.get().isValid()) { byte[] value = managedRocksIterator.get().value(); - CompactionLogEntry compactionLogEntry = - CompactionLogEntry.getFromProtobuf( - CompactionLogEntryProto.parseFrom(value)); - + CompactionLogEntry compactionLogEntry = CompactionLogEntry.getFromProtobuf( + CompactionLogEntryProto.parseFrom(value)); compactionLogEntry.getInputFileInfoList().forEach( f -> { String fileName = f.getFileName(); @@ -1965,7 +1964,7 @@ public void testPruneSSTFileValues() && filesInBackupDir.contains(fileName + SST_FILE_EXTENSION)) { assertTrue(f.isPruned()); try (ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( - new ManagedOptions(), file.getAbsolutePath(), 2 * 1024 * 1024); + new ManagedOptions(), file.getAbsolutePath(), 2 * 1024 * 1024); ManagedRawSSTFileIterator itr = sstFileReader.newIterator( keyValue -> keyValue.getValue(), null, null)) { while (itr.hasNext()) { @@ -1984,10 +1983,7 @@ public void testPruneSSTFileValues() private void createSSTFileWithKeys(Path dir, String fileName, int nKeys, int nTombstones) throws Exception { File file = Files.createFile(dir.resolve(fileName)).toFile(); - try (ManagedEnvOptions envOptions = new ManagedEnvOptions(); - ManagedOptions managedOptions = new ManagedOptions(); - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { - + try (ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(new ManagedEnvOptions(), new ManagedOptions())) { sstFileWriter.open(file.getAbsolutePath()); String generatedString = RandomStringUtils.randomAlphabetic(7); for (int n = 0; n < nKeys; n++) { @@ -2224,13 +2220,4 @@ public void testShouldSkipFile(String description, assertEquals(expectedResult, rocksDBCheckpointDiffer .shouldSkipCompaction(columnFamilyBytes, inputFiles, outputFiles)); } - - static class StripedLock implements StripedLockProvider { - - @Override - public Striped getStripedLock(String key) { - return SimpleStriped.readWriteLock(512, false); - } - - } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java index f6b6776882a6..ebe330c67ab1 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java @@ -19,12 +19,11 @@ import com.google.common.annotations.VisibleForTesting; import java.util.Collection; -import org.apache.hadoop.ozone.lock.StripedLockProvider; /** * Interface for OM Metadata locks. */ -public interface IOzoneManagerLock extends StripedLockProvider { +public interface IOzoneManagerLock { OMLockDetails acquireReadLock(OzoneManagerLock.Resource resource, String... resources); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java index b1c23d699e74..059536fe0a58 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java @@ -20,9 +20,7 @@ import static org.apache.hadoop.ozone.om.lock.OMLockDetails.EMPTY_DETAILS_LOCK_ACQUIRED; import static org.apache.hadoop.ozone.om.lock.OMLockDetails.EMPTY_DETAILS_LOCK_NOT_ACQUIRED; -import com.google.common.util.concurrent.Striped; import java.util.Collection; -import java.util.concurrent.locks.ReadWriteLock; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource; /** @@ -110,10 +108,4 @@ public OMLockMetrics getOMLockMetrics() { throw new UnsupportedOperationException( "OmReadOnlyLock does not support this operation."); } - - @Override - public Striped getStripedLock(String key) { - throw new UnsupportedOperationException( - "OmReadOnlyLock does not support this operation."); - } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java index 27738b8c0fd4..2f6d220e7afb 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java @@ -144,11 +144,6 @@ private ReentrantReadWriteLock getLock(Resource resource, String... keys) { return (ReentrantReadWriteLock) striped.get(key); } - @Override - public Striped getStripedLock(String resource) { - return stripedLockByResource.get(Resource.valueOf(resource)); - } - /** * Acquire read lock on resource. * @@ -591,8 +586,7 @@ public enum Resource { S3_SECRET_LOCK((byte) 4, "S3_SECRET_LOCK"), // 31 KEY_PATH_LOCK((byte) 5, "KEY_PATH_LOCK"), //63 PREFIX_LOCK((byte) 6, "PREFIX_LOCK"), //127 - SNAPSHOT_LOCK((byte) 7, "SNAPSHOT_LOCK"), // = 255 - SST_FILE_LOCK((byte) 8, "SST_FILE_LOCK"); // = 511 + SNAPSHOT_LOCK((byte) 7, "SNAPSHOT_LOCK"); // = 255 // level of the resource private byte lockLevel; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 63717b982588..b4d3afb4f2b8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -147,6 +147,7 @@ import org.apache.log4j.Logger; import org.apache.ozone.compaction.log.CompactionLogEntry; import org.apache.ozone.rocksdiff.CompactionNode; +import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Slow; import org.junit.jupiter.api.AfterAll; @@ -226,7 +227,7 @@ private void init() throws Exception { conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, KeyManagerImpl.DISABLE_VALUE); if (!disableNativeDiff) { - conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 1, TimeUnit.SECONDS); + conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 0, TimeUnit.SECONDS); } cluster = MiniOzoneCluster.newBuilder(conf) @@ -2491,30 +2492,28 @@ public void testSnapshotCompactionDag() throws Exception { null, 0).getDiffList().size()); if (!disableNativeDiff) { - // Verify backup SST files are pruned on DB compactions. - Thread.sleep(1000); - + // Prune SST files in compaction backup directory. RocksDatabase db = getRdbStore().getDb(); - java.nio.file.Path sstBackUpDir = java.nio.file.Paths.get(getRdbStore() - .getRocksDBCheckpointDiffer().getSSTBackupDir()); + RocksDBCheckpointDiffer differ = getRdbStore().getRocksDBCheckpointDiffer(); + differ.pruneSstFileValues(); - try (ManagedRocksIterator managedRocksIterator = new ManagedRocksIterator( - db.getManagedRocksDb().get().newIterator(db.getColumnFamily(COMPACTION_LOG_TABLE).getHandle()))) { + // Verify backup SST files are pruned on DB compactions. + java.nio.file.Path sstBackUpDir = java.nio.file.Paths.get(differ.getSSTBackupDir()); + try (ManagedOptions managedOptions = new ManagedOptions(); + ManagedRocksIterator managedRocksIterator = new ManagedRocksIterator( + db.getManagedRocksDb().get().newIterator(db.getColumnFamily(COMPACTION_LOG_TABLE).getHandle()))) { managedRocksIterator.get().seekToFirst(); while (managedRocksIterator.get().isValid()) { byte[] value = managedRocksIterator.get().value(); - CompactionLogEntry compactionLogEntry = - CompactionLogEntry.getFromProtobuf( - CompactionLogEntryProto.parseFrom(value)); - + CompactionLogEntry compactionLogEntry = CompactionLogEntry.getFromProtobuf( + CompactionLogEntryProto.parseFrom(value)); compactionLogEntry.getInputFileInfoList().forEach( f -> { java.nio.file.Path file = sstBackUpDir.resolve(f.getFileName() + ".sst"); - if (COLUMN_FAMILIES_TO_TRACK_IN_DAG.contains(f.getColumnFamily()) && - java.nio.file.Files.exists(file)) { + if (COLUMN_FAMILIES_TO_TRACK_IN_DAG.contains(f.getColumnFamily()) && java.nio.file.Files.exists(file)) { assertTrue(f.isPruned()); try (ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( - new ManagedOptions(), file.toFile().getAbsolutePath(), 2 * 1024 * 1024); + managedOptions, file.toFile().getAbsolutePath(), 2 * 1024 * 1024); ManagedRawSSTFileIterator itr = sstFileReader.newIterator( keyValue -> keyValue.getValue(), null, null)) { while (itr.hasNext()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 818bad249ddb..8a7b446c013c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -534,7 +534,7 @@ public void start(OzoneConfiguration configuration) throws IOException { int maxOpenFiles = configuration.getInt(OZONE_OM_DB_MAX_OPEN_FILES, OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT); - this.store = loadDB(configuration, metaDir, maxOpenFiles, getLock()); + this.store = loadDB(configuration, metaDir, maxOpenFiles); initializeOmTables(CacheType.FULL_CACHE, true); } @@ -542,15 +542,13 @@ public void start(OzoneConfiguration configuration) throws IOException { snapshotChainManager = new SnapshotChainManager(this); } - public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles, IOzoneManagerLock lock) - throws IOException { + public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles) throws IOException { return newDBStoreBuilder(configuration, null, metaDir) .setOpenReadOnly(false) .setEnableCompactionDag(true) .setCreateCheckpointDirs(true) .setEnableRocksDbMetrics(true) .setMaxNumberOfOpenFiles(maxOpenFiles) - .setLock(lock) .build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java index 263a389ffbda..395b37e859c3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.om.codec.OMDBDefinition; -import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -52,8 +51,7 @@ public void testDBDefinition() throws IOException { final int countOmDefTables = columnFamilyDefinitions.size(); List missingDBDefTables = new ArrayList<>(); - try (DBStore store = OmMetadataManagerImpl.loadDB(configuration, metaDir, -1, - new OzoneManagerLock(configuration))) { + try (DBStore store = OmMetadataManagerImpl.loadDB(configuration, metaDir, -1)) { // Get list of tables from the RocksDB Store final Collection missingOmDBTables = new ArrayList<>(store.getTableNames().values()); missingOmDBTables.remove("default"); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java index 602f3d10c98b..f85ec7099a92 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java @@ -48,7 +48,6 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; -import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.repair.RepairTool; import org.apache.ratis.util.Preconditions; @@ -503,9 +502,7 @@ protected static DBStore getStoreFromPath(String dbPath) throws IOException { "not exist or is not a RocksDB directory.", dbPath)); } // Load RocksDB and tables needed. - OzoneConfiguration conf = new OzoneConfiguration(); - return OmMetadataManagerImpl.loadDB(conf, new File(dbPath).getParentFile(), -1, - new OzoneManagerLock(conf)); + return OmMetadataManagerImpl.loadDB(new OzoneConfiguration(), new File(dbPath).getParentFile(), -1); } /** From cb90a23c9b43d5a42951b8557cbed68f4ee6c130 Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 30 Apr 2025 21:52:03 -0700 Subject: [PATCH 08/18] HDDS-12501. Fixed integration(snapshot) test. --- .../java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java | 2 +- .../main/java/org/apache/ozone/rocksdiff/CompactionNode.java | 1 + .../org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index 62d896d133c4..076b6bc933d0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -288,7 +288,7 @@ public DBStoreBuilder setEnableRocksDbMetrics(boolean enableRocksDbMetrics) { this.enableRocksDbMetrics = enableRocksDbMetrics; return this; } - + /** * Set the {@link ManagedDBOptions} and default * {@link ManagedColumnFamilyOptions} based on {@code prof}. diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java index 6054a55e608d..4e7c38c62c9a 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java @@ -38,6 +38,7 @@ public class CompactionNode { * @param numKeys Number of keys in the SST * @param seqNum Snapshot generation (sequence number) */ + public CompactionNode(String file, long numKeys, long seqNum, String startKey, String endKey, String columnFamily) { fileName = file; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index b4d3afb4f2b8..ddd7e60b0678 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -2524,6 +2524,7 @@ public void testSnapshotCompactionDag() throws Exception { assertFalse(f.isPruned()); } }); + managedRocksIterator.get().next(); } } } From d833f8a76469c155d035c36290bac745f843649f Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 7 May 2025 00:26:46 -0700 Subject: [PATCH 09/18] HDDS-12501. Simplified Exception handling. --- .../rocksdiff/RocksDBCheckpointDiffer.java | 93 +++++------ .../TestRocksDBCheckpointDiffer.java | 156 ++++++++++-------- 2 files changed, 128 insertions(+), 121 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index ed00c6d285d2..61ddb149449f 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -51,13 +51,14 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -72,6 +73,7 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.apache.hadoop.hdds.utils.Scheduler; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; @@ -184,9 +186,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private ColumnFamilyHandle compactionLogTableCFHandle; private ManagedRocksDB activeRocksDB; private ConcurrentMap inflightCompactions; - private boolean isNativeLibsLoaded = false; - private ConcurrentLinkedQueue> pruneQueue = - new ConcurrentLinkedQueue>(); + private Queue pruneQueue = null; /** * For snapshot diff calculation we only need to track following column @@ -271,13 +271,14 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, this.pruneSSTFileBatchSize = configuration.getInt( OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE, OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT); - if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { - try { - this.isNativeLibsLoaded = ManagedRawSSTFileReader.loadLibrary(); - } catch (NativeLibraryNotLoadedException e) { - LOG.warn("Native Library for raw sst file reading loading failed." + - " Cannot prune OMKeyInfo from SST files. {}", e.getMessage()); + try { + if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT) + && ManagedRawSSTFileReader.loadLibrary()) { + pruneQueue = new LinkedList<>(); } + } catch (NativeLibraryNotLoadedException e) { + LOG.warn("Native Library for raw sst file reading loading failed." + + " Cannot prune OMKeyInfo from SST files. {}", e.getMessage()); } if (pruneCompactionDagDaemonRunIntervalInMs > 0) { @@ -296,7 +297,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, pruneCompactionDagDaemonRunIntervalInMs, TimeUnit.MILLISECONDS); - if (isNativeLibsLoaded) { + if (pruneQueue != null) { this.scheduler.scheduleWithFixedDelay( this::pruneSstFileValues, pruneCompactionDagDaemonRunIntervalInMs, @@ -564,8 +565,8 @@ public void onCompactionCompleted(RocksDB db, } // Add the compaction log entry to the prune queue // so that the backup input sst files can be pruned. - if (isNativeLibsLoaded) { - pruneQueue.offer(Pair.of(key, compactionLogEntry)); + if (pruneQueue != null) { + pruneQueue.offer(key); } } }; @@ -812,8 +813,8 @@ public void loadAllCompactionLogs() { compactionLogEntry.getOutputFileInfoList(), compactionLogEntry.getDbSequenceNumber()); // Add the compaction log entry to the prune queue so that the backup input sst files can be pruned. - if (isNativeLibsLoaded) { - pruneQueue.offer(Pair.of(managedRocksIterator.get().key(), compactionLogEntry)); + if (pruneQueue != null) { + pruneQueue.offer(managedRocksIterator.get().key()); } managedRocksIterator.get().next(); } @@ -1438,16 +1439,21 @@ public void pruneSstFileValues() { } Path sstBackupDirPath = Paths.get(sstBackupDir); + Path prunedSSTFilePath = sstBackupDirPath.resolve("pruned.sst.tmp"); try (ManagedOptions managedOptions = new ManagedOptions(); ManagedEnvOptions envOptions = new ManagedEnvOptions(); ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { - Pair compactionLogTableEntry; + byte[] compactionLogEntryKey; int batchCounter = 0; - while ((compactionLogTableEntry = pruneQueue.poll()) != null && ++batchCounter <= pruneSSTFileBatchSize) { - byte[] key = compactionLogTableEntry.getKey(); - CompactionLogEntry compactionLogEntry = compactionLogTableEntry.getValue(); + while ((compactionLogEntryKey = pruneQueue.peek()) != null && ++batchCounter <= pruneSSTFileBatchSize) { + CompactionLogEntry compactionLogEntry; + try { + compactionLogEntry = CompactionLogEntry.getCodec().fromPersistedFormat( + activeRocksDB.get().get(compactionLogTableCFHandle, compactionLogEntryKey)); + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to get compaction log entry.", ex); + } boolean shouldUpdateTable = false; - boolean shouldReprocess = false; List fileInfoList = compactionLogEntry.getInputFileInfoList(); List updatedFileInfoList = new ArrayList<>(); for (CompactionFileInfo fileInfo : fileInfoList) { @@ -1455,21 +1461,19 @@ public void pruneSstFileValues() { updatedFileInfoList.add(fileInfo); continue; } - String sstFileName = fileInfo.getFileName(); - Path sstFilePath = sstBackupDirPath.resolve(sstFileName + ROCKSDB_SST_SUFFIX); + Path sstFilePath = sstBackupDirPath.resolve(fileInfo.getFileName() + ROCKSDB_SST_SUFFIX); if (Files.notExists(sstFilePath)) { LOG.debug("Skipping pruning SST file {} as it does not exist in backup directory.", sstFilePath); updatedFileInfoList.add(fileInfo); continue; } - // Write the file.sst => file.sst.tmp - Path prunedSSTFilePath = sstBackupDirPath.resolve(sstFilePath.toFile().getName() + ".tmp"); + // Write the file.sst => pruned.sst.tmp + Files.deleteIfExists(prunedSSTFilePath); try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( managedOptions, sstFilePath.toFile().getAbsolutePath(), SST_READ_AHEAD_SIZE); ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null)) { - Files.deleteIfExists(prunedSSTFilePath); sstFileWriter.open(prunedSSTFilePath.toFile().getAbsolutePath()); while (itr.hasNext()) { Pair keyValue = itr.next(); @@ -1479,17 +1483,13 @@ public void pruneSstFileValues() { sstFileWriter.put(keyValue.getKey(), new byte[0]); } } - } catch (IOException | RocksDBException e) { - shouldReprocess = true; - try { - Files.deleteIfExists(prunedSSTFilePath); - } catch (IOException ex) { } - continue; + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to write pruned entries for " + sstFilePath, ex); } finally { try { sstFileWriter.finish(); - } catch (RocksDBException e) { - throw new RuntimeException("Failed to finish writing to " + prunedSSTFilePath, e); + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to finish writing to " + prunedSSTFilePath, ex); } } @@ -1497,13 +1497,6 @@ public void pruneSstFileValues() { try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { Files.move(prunedSSTFilePath, sstFilePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException | InterruptedException e) { - shouldReprocess = true; - continue; - } finally { - try { - Files.deleteIfExists(prunedSSTFilePath); - } catch (IOException e) { } } shouldUpdateTable = true; fileInfo.setPruned(); @@ -1511,8 +1504,8 @@ public void pruneSstFileValues() { LOG.debug("Completed pruning OMKeyInfo from {}", sstFilePath); } + // Update Compaction Log table. Track keys that need updating. if (shouldUpdateTable) { - // Update Compaction Log table. Track keys that need updating. CompactionLogEntry.Builder builder = new CompactionLogEntry.Builder(compactionLogEntry.getDbSequenceNumber(), compactionLogEntry.getCompactionTime(), updatedFileInfoList, compactionLogEntry.getOutputFileInfoList()); String compactionReason = compactionLogEntry.getCompactionReason(); @@ -1521,19 +1514,19 @@ public void pruneSstFileValues() { } synchronized (this) { try { - activeRocksDB.get().put(compactionLogTableCFHandle, key, builder.build().getProtobuf().toByteArray()); - } catch (RocksDBException e) { - shouldReprocess = true; + activeRocksDB.get().put(compactionLogTableCFHandle, compactionLogEntryKey, + builder.build().getProtobuf().toByteArray()); + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to update the compaction log table for entry: " + + compactionLogEntry, ex); } } } - - if (shouldReprocess) { - LOG.error("Could not prune source files or update the compaction log table. Reprocessing entry: {}", - compactionLogEntry); - pruneQueue.offer(compactionLogTableEntry); - } + pruneQueue.poll(); } + Files.deleteIfExists(prunedSSTFilePath); + } catch (Exception e) { + LOG.error("Could not prune source OMKeyInfo from backup SST files.", e); } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index 162a77849897..3453890ad1fd 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -21,7 +21,6 @@ import static java.util.Arrays.asList; import static java.util.concurrent.TimeUnit.MINUTES; import static org.apache.hadoop.hdds.StringUtils.bytes2String; -import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; @@ -44,6 +43,7 @@ import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -55,7 +55,6 @@ import java.io.FileWriter; import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -65,6 +64,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -83,10 +83,11 @@ import java.util.stream.Stream; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.managed.ManagedCheckpoint; import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; @@ -95,9 +96,11 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; +import org.apache.hadoop.hdds.utils.db.managed.ManagedReadOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader; +import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReaderIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.util.Time; @@ -110,10 +113,10 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.rocksdb.ColumnFamilyDescriptor; @@ -361,11 +364,14 @@ public void init() throws RocksDBException { OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)).thenReturn(true); - rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer(METADATA_DIR_NAME, - SST_BACK_UP_DIR_NAME, - COMPACTION_LOG_DIR_NAME, - ACTIVE_DB_DIR_NAME, - config); + try (MockedStatic mockedRawSSTReader = Mockito.mockStatic(ManagedRawSSTFileReader.class)) { + mockedRawSSTReader.when(ManagedRawSSTFileReader::loadLibrary).thenReturn(true); + rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer(METADATA_DIR_NAME, + SST_BACK_UP_DIR_NAME, + COMPACTION_LOG_DIR_NAME, + ACTIVE_DB_DIR_NAME, + config); + } ManagedColumnFamilyOptions cfOpts = new ManagedColumnFamilyOptions(); cfOpts.optimizeUniversalStyleCompaction(); @@ -1913,87 +1919,95 @@ private static Stream casesGetSSTDiffListWithoutDB2() { /** * Test that backup SST files are pruned on loading previous compaction logs. */ - @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true") @Test - public void testPruneSSTFileValues() - throws Exception { - // Create src files in backup directory. - List filesInBackupDir = Arrays.asList("000078.sst", "000078.sst.tmp", "000073.sst"); - for (String fileName : filesInBackupDir) { - createSSTFileWithKeys(sstBackUpDir.toPath(), fileName, 1, 1); + public void testPruneSSTFileValues() throws Exception { + + List> keys = new ArrayList>(); + keys.add(Pair.of("key1".getBytes(UTF_8), Integer.valueOf(1))); + keys.add(Pair.of("key2".getBytes(UTF_8), Integer.valueOf(0))); + + // Create src files in backup & activedirectory. + // Pruning job should succeed when pruned temp file is already present. + List filesInBackupDir = Arrays.asList( + sstBackUpDir + "/000078.sst", sstBackUpDir + "/pruned.sst.tmp", sstBackUpDir + "/000073.sst"); + for (String f : filesInBackupDir) { + createSSTFileWithKeys(f, keys); } - - // Create compacted files in active DB directory. - createSSTFileWithKeys(activeDbDir.toPath(), "000081.sst", 1, 1); + createSSTFileWithKeys(activeDbDir + "/000081.sst", keys); // Load dummy previous compaction logs. CompactionLogEntry dummyEntry = new CompactionLogEntry(178, System.currentTimeMillis(), - Arrays.asList(new CompactionFileInfo("000078", - "/volume/bucket1/key-0000001411", - "/volume/bucket2/key-0000099649", - "keyTable"), - new CompactionFileInfo("000073", - "/volume/bucket1/key-0000000730", - "/volume/bucket2/key-0000097010", - "keyTable")), - Collections.singletonList(new CompactionFileInfo("000081", - "/volume/bucket1/key-0000000730", - "/volume/bucket2/key-0000099649", - "keyTable")), + Arrays.asList(new CompactionFileInfo("000078", "/volume/bucket1/key-0000001411", + "/volume/bucket2/key-0000099649", "keyTable"), + new CompactionFileInfo("000073", "/volume/bucket1/key-0000000730", + "/volume/bucket2/key-0000097010", "keyTable")), + Collections.singletonList(new CompactionFileInfo("000081", "/volume/bucket1/key-0000000730", + "/volume/bucket2/key-0000099649", "keyTable")), null ); - rocksDBCheckpointDiffer.addToCompactionLogTable(dummyEntry); + byte[] dummyEntryKey = rocksDBCheckpointDiffer.addToCompactionLogTable(dummyEntry); rocksDBCheckpointDiffer.loadAllCompactionLogs(); - // Run the SST file pruner. - waitForLock(rocksDBCheckpointDiffer, RocksDBCheckpointDiffer::pruneSstFileValues); + // Pruning should succeed even if a source SST file is missing. + Files.delete(sstBackUpDir.toPath().resolve("000073.sst")); + ManagedRawSSTFileIterator mockedRawSSTFileItr = mock(ManagedRawSSTFileIterator.class); + when(mockedRawSSTFileItr.hasNext()).thenReturn(true, true, false); + when(mockedRawSSTFileItr.next()).thenReturn(keys.get(0), keys.get(1)); + try (MockedConstruction mockedRawSSTReader = Mockito.mockConstruction( + ManagedRawSSTFileReader.class, (mock, context) -> { + when(mock.newIterator(any(), any(), any())).thenReturn(mockedRawSSTFileItr); + doNothing().when(mock).close(); + })) { + // Run the SST file pruner. + waitForLock(rocksDBCheckpointDiffer, RocksDBCheckpointDiffer::pruneSstFileValues); + } + assertFalse(Files.exists(sstBackUpDir.toPath().resolve("pruned.sst.tmp"))); // Verify that files are pruned. - try (ManagedRocksIterator managedRocksIterator = new ManagedRocksIterator( - activeRocksDB.get().newIterator(compactionLogTableCFHandle))) { - managedRocksIterator.get().seekToFirst(); - while (managedRocksIterator.get().isValid()) { - byte[] value = managedRocksIterator.get().value(); - CompactionLogEntry compactionLogEntry = CompactionLogEntry.getFromProtobuf( - CompactionLogEntryProto.parseFrom(value)); - compactionLogEntry.getInputFileInfoList().forEach( - f -> { - String fileName = f.getFileName(); - File file = sstBackUpDir.toPath().resolve(fileName + SST_FILE_EXTENSION).toFile(); - if (COLUMN_FAMILIES_TO_TRACK_IN_DAG.contains(f.getColumnFamily()) - && filesInBackupDir.contains(fileName + SST_FILE_EXTENSION)) { - assertTrue(f.isPruned()); - try (ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>( - new ManagedOptions(), file.getAbsolutePath(), 2 * 1024 * 1024); - ManagedRawSSTFileIterator itr = sstFileReader.newIterator( - keyValue -> keyValue.getValue(), null, null)) { - while (itr.hasNext()) { - assertEquals(0, itr.next().length); - } - } - } else { - assertFalse(f.isPruned()); - } - }); - managedRocksIterator.get().next(); + CompactionLogEntry compactionLogEntry; + try { + compactionLogEntry = CompactionLogEntry.getCodec().fromPersistedFormat( + activeRocksDB.get().get(compactionLogTableCFHandle, dummyEntryKey)); + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to get compaction log entry.", ex); + } + for (CompactionFileInfo fileInfo : compactionLogEntry.getInputFileInfoList()) { + if (fileInfo.getFileName().equals("000078")) { + assertTrue(fileInfo.isPruned()); + File file = sstBackUpDir.toPath().resolve(fileInfo.getFileName() + SST_FILE_EXTENSION).toFile(); + ManagedSstFileReader sstFileReader = new ManagedSstFileReader(new ManagedOptions()); + sstFileReader.open(file.getAbsolutePath()); + ManagedSstFileReaderIterator itr = ManagedSstFileReaderIterator + .managed(sstFileReader.newIterator(new ManagedReadOptions())); + itr.get().seekToFirst(); + while (itr.get().isValid()) { + assertEquals(0, itr.get().value().length); + itr.get().next(); + } + itr.close(); + sstFileReader.close(); + } else { + assertFalse(fileInfo.isPruned()); } } } - private void createSSTFileWithKeys(Path dir, String fileName, int nKeys, int nTombstones) + private void createSSTFileWithKeys(String filePath, List> keys) throws Exception { - File file = Files.createFile(dir.resolve(fileName)).toFile(); try (ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(new ManagedEnvOptions(), new ManagedOptions())) { - sstFileWriter.open(file.getAbsolutePath()); - String generatedString = RandomStringUtils.randomAlphabetic(7); - for (int n = 0; n < nKeys; n++) { - sstFileWriter.put(("key" + n + "-" + generatedString).getBytes(StandardCharsets.UTF_8), - ("value" + n + "-" + generatedString).getBytes(StandardCharsets.UTF_8)); - } - for (int n = nKeys; n < nKeys + nTombstones; n++) { - sstFileWriter.delete(("key" + n + "-" + generatedString).getBytes(StandardCharsets.UTF_8)); + sstFileWriter.open(filePath); + Iterator> itr = keys.iterator(); + while (itr.hasNext()) { + Pair entry = itr.next(); + if (entry.getValue() == 0) { + sstFileWriter.delete(entry.getKey()); + } else { + sstFileWriter.put(entry.getKey(), "dummyValue".getBytes(UTF_8)); + } } sstFileWriter.finish(); + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to get write " + filePath, ex); } } From e2f88218460d494abe8b19d811c06bf678099198 Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 7 May 2025 00:56:13 -0700 Subject: [PATCH 10/18] HDDS-12501. Fixed findbugs and compilation issue. --- .../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 14 +++++++------- .../ozone/om/snapshot/SnapshotDiffManager.java | 2 -- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 4eaf2c5afd0b..fd96a4e3a01e 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -78,7 +78,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; -import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.ozone.compaction.log.CompactionFileInfo; @@ -180,7 +179,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final ConcurrentMap inflightCompactions; private Queue pruneQueue = null; - /** * For snapshot diff calculation we only need to track following column * families. Other column families are irrelevant for snapshot diff. @@ -1268,11 +1266,13 @@ public void pruneSstFileValues() { int batchCounter = 0; while ((compactionLogEntryKey = pruneQueue.peek()) != null && ++batchCounter <= pruneSSTFileBatchSize) { CompactionLogEntry compactionLogEntry; - try { - compactionLogEntry = CompactionLogEntry.getCodec().fromPersistedFormat( - activeRocksDB.get().get(compactionLogTableCFHandle, compactionLogEntryKey)); - } catch (RocksDBException ex) { - throw new RocksDatabaseException("Failed to get compaction log entry.", ex); + synchronized (this) { + try { + compactionLogEntry = CompactionLogEntry.getCodec().fromPersistedFormat( + activeRocksDB.get().get(compactionLogTableCFHandle, compactionLogEntryKey)); + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to get compaction log entry.", ex); + } } boolean shouldUpdateTable = false; List fileInfoList = compactionLogEntry.getInputFileInfoList(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 0219b5a58cbb..60acfc81a328 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -36,8 +36,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_THREAD_POOL_SIZE_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE; From b11fe0a9798ddedf25e7fe1ef31027ef34a04d53 Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 7 May 2025 12:41:37 -0700 Subject: [PATCH 11/18] HDDS-12501. Fixed unit test. --- .../rocksdiff/RocksDBCheckpointDiffer.java | 5 +- .../TestRocksDBCheckpointDiffer.java | 108 ++++++++++-------- 2 files changed, 61 insertions(+), 52 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index fd96a4e3a01e..1fb8e86bb660 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -153,6 +153,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, static final String SST_FILE_EXTENSION = ".sst"; public static final int SST_FILE_EXTENSION_LENGTH = SST_FILE_EXTENSION.length(); + static final String PRUNED_SST_FILE_TEMP = "pruned.sst.tmp"; private static final int LONG_MAX_STR_LEN = String.valueOf(Long.MAX_VALUE).length(); @@ -1258,7 +1259,7 @@ public void pruneSstFileValues() { } Path sstBackupDirPath = Paths.get(sstBackupDir); - Path prunedSSTFilePath = sstBackupDirPath.resolve("pruned.sst.tmp"); + Path prunedSSTFilePath = sstBackupDirPath.resolve(PRUNED_SST_FILE_TEMP); try (ManagedOptions managedOptions = new ManagedOptions(); ManagedEnvOptions envOptions = new ManagedEnvOptions(); ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { @@ -1346,7 +1347,7 @@ public void pruneSstFileValues() { pruneQueue.poll(); } Files.deleteIfExists(prunedSSTFilePath); - } catch (Exception e) { + } catch (IOException | InterruptedException e) { LOG.error("Could not prune source OMKeyInfo from backup SST files.", e); } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index a23c6a9d0e75..c421a8a68815 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -32,6 +32,7 @@ import static org.apache.hadoop.util.Time.now; import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG; import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COMPACTION_LOG_FILE_NAME_SUFFIX; +import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.PRUNED_SST_FILE_TEMP; import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.SST_FILE_EXTENSION; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -1939,71 +1940,78 @@ public void testPruneSSTFileValues() throws Exception { List> keys = new ArrayList>(); keys.add(Pair.of("key1".getBytes(UTF_8), Integer.valueOf(1))); keys.add(Pair.of("key2".getBytes(UTF_8), Integer.valueOf(0))); + keys.add(Pair.of("key3".getBytes(UTF_8), Integer.valueOf(1))); - // Create src files in backup & activedirectory. + String inputFile78 = "000078"; + String inputFile73 = "000073"; + String outputFile81 = "000081"; + // Create src & destination files in backup & activedirectory. // Pruning job should succeed when pruned temp file is already present. - List filesInBackupDir = Arrays.asList( - sstBackUpDir + "/000078.sst", sstBackUpDir + "/pruned.sst.tmp", sstBackUpDir + "/000073.sst"); - for (String f : filesInBackupDir) { - createSSTFileWithKeys(f, keys); - } - createSSTFileWithKeys(activeDbDir + "/000081.sst", keys); - - // Load dummy previous compaction logs. - CompactionLogEntry dummyEntry = new CompactionLogEntry(178, System.currentTimeMillis(), - Arrays.asList(new CompactionFileInfo("000078", "/volume/bucket1/key-0000001411", - "/volume/bucket2/key-0000099649", "keyTable"), - new CompactionFileInfo("000073", "/volume/bucket1/key-0000000730", - "/volume/bucket2/key-0000097010", "keyTable")), - Collections.singletonList(new CompactionFileInfo("000081", "/volume/bucket1/key-0000000730", - "/volume/bucket2/key-0000099649", "keyTable")), + createSSTFileWithKeys(sstBackUpDir + "/" + inputFile78 + SST_FILE_EXTENSION, keys); + createSSTFileWithKeys(sstBackUpDir + "/" + inputFile73 + SST_FILE_EXTENSION, keys); + createSSTFileWithKeys(sstBackUpDir + PRUNED_SST_FILE_TEMP, keys); + createSSTFileWithKeys(activeDbDir + "/" + outputFile81 + SST_FILE_EXTENSION, keys); + + // Load compaction log + CompactionLogEntry compactionLogEntry = new CompactionLogEntry(178, System.currentTimeMillis(), + Arrays.asList( + new CompactionFileInfo(inputFile78, "/volume/bucket1/key-5", "/volume/bucket2/key-10", "keyTable"), + new CompactionFileInfo(inputFile73, "/volume/bucket1/key-1", "/volume/bucket2/key-5", "keyTable")), + Collections.singletonList( + new CompactionFileInfo(outputFile81, "/volume/bucket1/key-1", "/volume/bucket2/key-10", "keyTable")), null ); - byte[] dummyEntryKey = rocksDBCheckpointDiffer.addToCompactionLogTable(dummyEntry); + byte[] compactionLogEntryKey = rocksDBCheckpointDiffer.addToCompactionLogTable(compactionLogEntry); rocksDBCheckpointDiffer.loadAllCompactionLogs(); - // Pruning should succeed even if a source SST file is missing. - Files.delete(sstBackUpDir.toPath().resolve("000073.sst")); + // Pruning should not fail a source SST file has been removed by a another pruner. + Files.delete(sstBackUpDir.toPath().resolve(inputFile73 + SST_FILE_EXTENSION)); + // Run the SST file pruner. ManagedRawSSTFileIterator mockedRawSSTFileItr = mock(ManagedRawSSTFileIterator.class); - when(mockedRawSSTFileItr.hasNext()).thenReturn(true, true, false); - when(mockedRawSSTFileItr.next()).thenReturn(keys.get(0), keys.get(1)); + Iterator keyItr = keys.iterator(); + when(mockedRawSSTFileItr.hasNext()).thenReturn(true, true, true, false); + when(mockedRawSSTFileItr.next()).thenReturn(keyItr.next(), keyItr.next(), keyItr.next()); try (MockedConstruction mockedRawSSTReader = Mockito.mockConstruction( - ManagedRawSSTFileReader.class, (mock, context) -> { - when(mock.newIterator(any(), any(), any())).thenReturn(mockedRawSSTFileItr); - doNothing().when(mock).close(); - })) { - // Run the SST file pruner. - waitForLock(rocksDBCheckpointDiffer, RocksDBCheckpointDiffer::pruneSstFileValues); + ManagedRawSSTFileReader.class, (mock, context) -> { + when(mock.newIterator(any(), any(), any())).thenReturn(mockedRawSSTFileItr); + doNothing().when(mock).close(); + })) { + rocksDBCheckpointDiffer.pruneSstFileValues(); } - assertFalse(Files.exists(sstBackUpDir.toPath().resolve("pruned.sst.tmp"))); + // pruned.sst.tmp should be deleted when pruning job exits successfully. + assertFalse(Files.exists(sstBackUpDir.toPath().resolve(PRUNED_SST_FILE_TEMP))); - // Verify that files are pruned. - CompactionLogEntry compactionLogEntry; + CompactionLogEntry updatedLogEntry; try { - compactionLogEntry = CompactionLogEntry.getCodec().fromPersistedFormat( - activeRocksDB.get().get(compactionLogTableCFHandle, dummyEntryKey)); + updatedLogEntry = CompactionLogEntry.getCodec().fromPersistedFormat( + activeRocksDB.get().get(compactionLogTableCFHandle, compactionLogEntryKey)); } catch (RocksDBException ex) { throw new RocksDatabaseException("Failed to get compaction log entry.", ex); } - for (CompactionFileInfo fileInfo : compactionLogEntry.getInputFileInfoList()) { - if (fileInfo.getFileName().equals("000078")) { - assertTrue(fileInfo.isPruned()); - File file = sstBackUpDir.toPath().resolve(fileInfo.getFileName() + SST_FILE_EXTENSION).toFile(); - ManagedSstFileReader sstFileReader = new ManagedSstFileReader(new ManagedOptions()); - sstFileReader.open(file.getAbsolutePath()); - ManagedSstFileReaderIterator itr = ManagedSstFileReaderIterator - .managed(sstFileReader.newIterator(new ManagedReadOptions())); - itr.get().seekToFirst(); - while (itr.get().isValid()) { - assertEquals(0, itr.get().value().length); - itr.get().next(); - } - itr.close(); - sstFileReader.close(); - } else { - assertFalse(fileInfo.isPruned()); - } + CompactionFileInfo fileInfo78 = updatedLogEntry.getInputFileInfoList().get(0); + CompactionFileInfo fileInfo73 = updatedLogEntry.getInputFileInfoList().get(1); + + // Verify 000078.sst has been pruned + assertTrue(fileInfo78.getFileName().equals(inputFile78)); + assertTrue(fileInfo78.isPruned()); + ManagedSstFileReader sstFileReader = new ManagedSstFileReader(new ManagedOptions()); + sstFileReader.open(sstBackUpDir.toPath().resolve(inputFile78 + SST_FILE_EXTENSION).toFile().getAbsolutePath()); + ManagedSstFileReaderIterator itr = ManagedSstFileReaderIterator + .managed(sstFileReader.newIterator(new ManagedReadOptions())); + itr.get().seekToFirst(); + int prunedKeys = 0; + while (itr.get().isValid()) { + // Verify that value is removed for non-tombstone keys. + assertEquals(0, itr.get().value().length); + prunedKeys++; + itr.get().next(); } + assertEquals(2, prunedKeys); + itr.close(); + sstFileReader.close(); + + // Verify 000073.sst pruning has been skipped + assertFalse(fileInfo73.isPruned()); } private void createSSTFileWithKeys(String filePath, List> keys) From dbea3583996e40053a90795b1760e909a2975a2b Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 7 May 2025 12:48:14 -0700 Subject: [PATCH 12/18] HDDS-12501. findbugs fixes. --- .../org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index c421a8a68815..d3ee925522fd 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -1992,7 +1992,7 @@ public void testPruneSSTFileValues() throws Exception { CompactionFileInfo fileInfo73 = updatedLogEntry.getInputFileInfoList().get(1); // Verify 000078.sst has been pruned - assertTrue(fileInfo78.getFileName().equals(inputFile78)); + assertEquals(inputFile78, fileInfo78.getFileName()); assertTrue(fileInfo78.isPruned()); ManagedSstFileReader sstFileReader = new ManagedSstFileReader(new ManagedOptions()); sstFileReader.open(sstBackUpDir.toPath().resolve(inputFile78 + SST_FILE_EXTENSION).toFile().getAbsolutePath()); From 22728372325b73f3206f5ce08ad363022671193e Mon Sep 17 00:00:00 2001 From: saketa Date: Thu, 8 May 2025 19:31:27 -0700 Subject: [PATCH 13/18] HDDS-12501. Addressed comments. --- .../rocksdiff/RocksDBCheckpointDiffer.java | 114 +++++++++--------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 1fb8e86bb660..7279937b0a1e 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -49,13 +49,13 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -241,7 +241,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, try { if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT) && ManagedRawSSTFileReader.loadLibrary()) { - pruneQueue = new LinkedList<>(); + pruneQueue = new ConcurrentLinkedQueue<>(); } } catch (NativeLibraryNotLoadedException e) { LOG.warn("Native Library for raw sst file reading loading failed." + @@ -1274,67 +1274,68 @@ public void pruneSstFileValues() { } catch (RocksDBException ex) { throw new RocksDatabaseException("Failed to get compaction log entry.", ex); } - } - boolean shouldUpdateTable = false; - List fileInfoList = compactionLogEntry.getInputFileInfoList(); - List updatedFileInfoList = new ArrayList<>(); - for (CompactionFileInfo fileInfo : fileInfoList) { - if (fileInfo.isPruned()) { - updatedFileInfoList.add(fileInfo); - continue; - } - Path sstFilePath = sstBackupDirPath.resolve(fileInfo.getFileName() + ROCKSDB_SST_SUFFIX); - if (Files.notExists(sstFilePath)) { - LOG.debug("Skipping pruning SST file {} as it does not exist in backup directory.", sstFilePath); - updatedFileInfoList.add(fileInfo); - continue; - } - // Write the file.sst => pruned.sst.tmp - Files.deleteIfExists(prunedSSTFilePath); - try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( - managedOptions, sstFilePath.toFile().getAbsolutePath(), SST_READ_AHEAD_SIZE); - ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( - keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null)) { - sstFileWriter.open(prunedSSTFilePath.toFile().getAbsolutePath()); - while (itr.hasNext()) { - Pair keyValue = itr.next(); - if (keyValue.getValue() == 0) { - sstFileWriter.delete(keyValue.getKey()); - } else { - sstFileWriter.put(keyValue.getKey(), new byte[0]); - } + boolean shouldUpdateTable = false; + List fileInfoList = compactionLogEntry.getInputFileInfoList(); + List updatedFileInfoList = new ArrayList<>(); + for (CompactionFileInfo fileInfo : fileInfoList) { + if (fileInfo.isPruned()) { + updatedFileInfoList.add(fileInfo); + continue; } - } catch (RocksDBException ex) { - throw new RocksDatabaseException("Failed to write pruned entries for " + sstFilePath, ex); - } finally { - try { - sstFileWriter.finish(); + Path sstFilePath = sstBackupDirPath.resolve(fileInfo.getFileName() + ROCKSDB_SST_SUFFIX); + if (Files.notExists(sstFilePath)) { + LOG.debug("Skipping pruning SST file {} as it does not exist in backup directory.", sstFilePath); + updatedFileInfoList.add(fileInfo); + continue; + } + + // Write the file.sst => pruned.sst.tmp + Files.deleteIfExists(prunedSSTFilePath); + try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( + managedOptions, sstFilePath.toFile().getAbsolutePath(), SST_READ_AHEAD_SIZE); + ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( + keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null)) { + sstFileWriter.open(prunedSSTFilePath.toFile().getAbsolutePath()); + while (itr.hasNext()) { + Pair keyValue = itr.next(); + if (keyValue.getValue() == 0) { + sstFileWriter.delete(keyValue.getKey()); + } else { + sstFileWriter.put(keyValue.getKey(), new byte[0]); + } + } } catch (RocksDBException ex) { - throw new RocksDatabaseException("Failed to finish writing to " + prunedSSTFilePath, ex); + throw new RocksDatabaseException("Failed to write pruned entries for " + sstFilePath, ex); + } finally { + try { + sstFileWriter.finish(); + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to close SSTFileWriter writing to path: " + + prunedSSTFilePath, ex); + } } - } - // Move file.sst.tmp to file.sst and replace existing file atomically - try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { - Files.move(prunedSSTFilePath, sstFilePath, - StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + // Move file.sst.tmp to file.sst and replace existing file atomically + try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { + Files.move(prunedSSTFilePath, sstFilePath, + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + shouldUpdateTable = true; + fileInfo.setPruned(); + updatedFileInfoList.add(fileInfo); + LOG.debug("Completed pruning OMKeyInfo from {}", sstFilePath); } - shouldUpdateTable = true; - fileInfo.setPruned(); - updatedFileInfoList.add(fileInfo); - LOG.debug("Completed pruning OMKeyInfo from {}", sstFilePath); - } - // Update Compaction Log table. Track keys that need updating. - if (shouldUpdateTable) { - CompactionLogEntry.Builder builder = new CompactionLogEntry.Builder(compactionLogEntry.getDbSequenceNumber(), - compactionLogEntry.getCompactionTime(), updatedFileInfoList, compactionLogEntry.getOutputFileInfoList()); - String compactionReason = compactionLogEntry.getCompactionReason(); - if (compactionReason != null) { - builder.setCompactionReason(compactionReason); - } - synchronized (this) { + // Update Compaction Log table. Track keys that need updating. + if (shouldUpdateTable) { + CompactionLogEntry.Builder builder = new CompactionLogEntry.Builder( + compactionLogEntry.getDbSequenceNumber(), compactionLogEntry.getCompactionTime(), + updatedFileInfoList, compactionLogEntry.getOutputFileInfoList()); + String compactionReason = compactionLogEntry.getCompactionReason(); + if (compactionReason != null) { + builder.setCompactionReason(compactionReason); + } try { activeRocksDB.get().put(compactionLogTableCFHandle, compactionLogEntryKey, builder.build().getProtobuf().toByteArray()); @@ -1346,7 +1347,6 @@ public void pruneSstFileValues() { } pruneQueue.poll(); } - Files.deleteIfExists(prunedSSTFilePath); } catch (IOException | InterruptedException e) { LOG.error("Could not prune source OMKeyInfo from backup SST files.", e); } From 13e646590b1d064d3c67d1fd39f2b3d66be935cb Mon Sep 17 00:00:00 2001 From: saketa Date: Mon, 12 May 2025 12:12:42 -0700 Subject: [PATCH 14/18] HDDS-12501. Moved pruning to a separate function. Added CompactionLogEntry.toBuilder(). --- .../compaction/log/CompactionLogEntry.java | 17 ++++- .../rocksdiff/RocksDBCheckpointDiffer.java | 71 ++++++++++--------- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java index 8ee48e43f903..e209420e9545 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java @@ -129,13 +129,23 @@ public String toString() { inputFileInfoList, outputFileInfoList, compactionReason); } + public static Builder toBuilder(CompactionLogEntry compactionLogEntry) { + Builder builder = new Builder(compactionLogEntry.getDbSequenceNumber(), compactionLogEntry.getCompactionTime(), + compactionLogEntry.getInputFileInfoList(), compactionLogEntry.getOutputFileInfoList()); + String compactionReason = compactionLogEntry.getCompactionReason(); + if (compactionLogEntry.getCompactionReason() != null) { + builder.setCompactionReason(compactionReason); + } + return builder; + } + /** * Builder of CompactionLogEntry. */ public static class Builder { private final long dbSequenceNumber; private final long compactionTime; - private final List inputFileInfoList; + private List inputFileInfoList; private final List outputFileInfoList; private String compactionReason; @@ -157,6 +167,11 @@ public Builder setCompactionReason(String compactionReason) { return this; } + public Builder updateInputFileInoList(List fileInfoList) { + this.inputFileInfoList = fileInfoList; + return this; + } + public CompactionLogEntry build() { return new CompactionLogEntry(dbSequenceNumber, compactionTime, inputFileInfoList, outputFileInfoList, compactionReason); diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 7279937b0a1e..beb39e3528b2 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -18,6 +18,7 @@ package org.apache.ozone.rocksdiff; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; @@ -1261,12 +1262,12 @@ public void pruneSstFileValues() { Path sstBackupDirPath = Paths.get(sstBackupDir); Path prunedSSTFilePath = sstBackupDirPath.resolve(PRUNED_SST_FILE_TEMP); try (ManagedOptions managedOptions = new ManagedOptions(); - ManagedEnvOptions envOptions = new ManagedEnvOptions(); - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { + ManagedEnvOptions envOptions = new ManagedEnvOptions()) { byte[] compactionLogEntryKey; int batchCounter = 0; while ((compactionLogEntryKey = pruneQueue.peek()) != null && ++batchCounter <= pruneSSTFileBatchSize) { CompactionLogEntry compactionLogEntry; + // Get the compaction log entry. synchronized (this) { try { compactionLogEntry = CompactionLogEntry.getCodec().fromPersistedFormat( @@ -1279,6 +1280,7 @@ public void pruneSstFileValues() { List fileInfoList = compactionLogEntry.getInputFileInfoList(); List updatedFileInfoList = new ArrayList<>(); for (CompactionFileInfo fileInfo : fileInfoList) { + // Skip pruning file if it is already pruned or is removed. if (fileInfo.isPruned()) { updatedFileInfoList.add(fileInfo); continue; @@ -1290,33 +1292,12 @@ public void pruneSstFileValues() { continue; } - // Write the file.sst => pruned.sst.tmp + // Prune file.sst => pruned.sst.tmp Files.deleteIfExists(prunedSSTFilePath); - try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( - managedOptions, sstFilePath.toFile().getAbsolutePath(), SST_READ_AHEAD_SIZE); - ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( - keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null)) { - sstFileWriter.open(prunedSSTFilePath.toFile().getAbsolutePath()); - while (itr.hasNext()) { - Pair keyValue = itr.next(); - if (keyValue.getValue() == 0) { - sstFileWriter.delete(keyValue.getKey()); - } else { - sstFileWriter.put(keyValue.getKey(), new byte[0]); - } - } - } catch (RocksDBException ex) { - throw new RocksDatabaseException("Failed to write pruned entries for " + sstFilePath, ex); - } finally { - try { - sstFileWriter.finish(); - } catch (RocksDBException ex) { - throw new RocksDatabaseException("Failed to close SSTFileWriter writing to path: " - + prunedSSTFilePath, ex); - } - } + removeValueFromSSTFile(managedOptions, envOptions, sstFilePath.toFile().getAbsolutePath(), + prunedSSTFilePath.toFile().getAbsolutePath()); - // Move file.sst.tmp to file.sst and replace existing file atomically + // Move pruned.sst.tmp => file.sst and replace existing file atomically. try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { Files.move(prunedSSTFilePath, sstFilePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); @@ -1327,15 +1308,10 @@ public void pruneSstFileValues() { LOG.debug("Completed pruning OMKeyInfo from {}", sstFilePath); } - // Update Compaction Log table. Track keys that need updating. + // Update compaction log entry in table. if (shouldUpdateTable) { - CompactionLogEntry.Builder builder = new CompactionLogEntry.Builder( - compactionLogEntry.getDbSequenceNumber(), compactionLogEntry.getCompactionTime(), - updatedFileInfoList, compactionLogEntry.getOutputFileInfoList()); - String compactionReason = compactionLogEntry.getCompactionReason(); - if (compactionReason != null) { - builder.setCompactionReason(compactionReason); - } + CompactionLogEntry.Builder builder = CompactionLogEntry.toBuilder(compactionLogEntry); + builder.updateInputFileInoList(updatedFileInfoList); try { activeRocksDB.get().put(compactionLogTableCFHandle, compactionLogEntryKey, builder.build().getProtobuf().toByteArray()); @@ -1352,6 +1328,31 @@ public void pruneSstFileValues() { } } + private void removeValueFromSSTFile(ManagedOptions options, ManagedEnvOptions envOptions, + String sstFilePath, String prunedFilePath) + throws IOException { + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, options); + try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( + options, sstFilePath, SST_READ_AHEAD_SIZE); + ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( + keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null)) { + sstFileWriter.open(prunedFilePath); + while (itr.hasNext()) { + Pair keyValue = itr.next(); + if (keyValue.getValue() == 0) { + sstFileWriter.delete(keyValue.getKey()); + } else { + sstFileWriter.put(keyValue.getKey(), EMPTY_BYTE_ARRAY); + } + } + sstFileWriter.finish(); + } catch (RocksDBException ex) { + throw new RocksDatabaseException("Failed to write pruned entries for " + sstFilePath, ex); + } finally { + sstFileWriter.close(); + } + } + public boolean shouldRun() { return !suspended.get(); } From 97ac54f1f6ae013c8e77a1c465604a2a68cf955b Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Mon, 12 May 2025 14:34:04 -0700 Subject: [PATCH 15/18] Apply suggestions from code review --- .../ozone/compaction/log/CompactionLogEntry.java | 2 +- .../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java index e209420e9545..49ccfae15fec 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java @@ -167,7 +167,7 @@ public Builder setCompactionReason(String compactionReason) { return this; } - public Builder updateInputFileInoList(List fileInfoList) { + public Builder updateInputFileInfoList(List fileInfoList) { this.inputFileInfoList = fileInfoList; return this; } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index beb39e3528b2..d8ebfb164c50 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -1310,8 +1310,8 @@ public void pruneSstFileValues() { // Update compaction log entry in table. if (shouldUpdateTable) { - CompactionLogEntry.Builder builder = CompactionLogEntry.toBuilder(compactionLogEntry); - builder.updateInputFileInoList(updatedFileInfoList); + CompactionLogEntry.Builder builder = compactionLogEntry.toBuilder(); + builder.updateInputFileInfoList(updatedFileInfoList); try { activeRocksDB.get().put(compactionLogTableCFHandle, compactionLogEntryKey, builder.build().getProtobuf().toByteArray()); @@ -1331,11 +1331,11 @@ public void pruneSstFileValues() { private void removeValueFromSSTFile(ManagedOptions options, ManagedEnvOptions envOptions, String sstFilePath, String prunedFilePath) throws IOException { - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, options); try (ManagedRawSSTFileReader> sstFileReader = new ManagedRawSSTFileReader<>( options, sstFilePath, SST_READ_AHEAD_SIZE); ManagedRawSSTFileIterator> itr = sstFileReader.newIterator( - keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null)) { + keyValue -> Pair.of(keyValue.getKey(), keyValue.getType()), null, null); + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, options);) { sstFileWriter.open(prunedFilePath); while (itr.hasNext()) { Pair keyValue = itr.next(); @@ -1348,8 +1348,6 @@ private void removeValueFromSSTFile(ManagedOptions options, ManagedEnvOptions en sstFileWriter.finish(); } catch (RocksDBException ex) { throw new RocksDatabaseException("Failed to write pruned entries for " + sstFilePath, ex); - } finally { - sstFileWriter.close(); } } From e1487b618ba5204b93c44162289ba868689e918b Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Mon, 12 May 2025 14:35:50 -0700 Subject: [PATCH 16/18] Update toBuilder() Co-authored-by: Swaminathan Balachandran <47532440+swamirishi@users.noreply.github.com> --- .../apache/ozone/compaction/log/CompactionLogEntry.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java index 49ccfae15fec..83b456e1e767 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java @@ -129,10 +129,10 @@ public String toString() { inputFileInfoList, outputFileInfoList, compactionReason); } - public static Builder toBuilder(CompactionLogEntry compactionLogEntry) { - Builder builder = new Builder(compactionLogEntry.getDbSequenceNumber(), compactionLogEntry.getCompactionTime(), - compactionLogEntry.getInputFileInfoList(), compactionLogEntry.getOutputFileInfoList()); - String compactionReason = compactionLogEntry.getCompactionReason(); + public Builder toBuilder() { + Builder builder = new Builder(this.getDbSequenceNumber(), this.getCompactionTime(), + this.getInputFileInfoList(), this.getOutputFileInfoList()); + String compactionReason = this.getCompactionReason(); if (compactionLogEntry.getCompactionReason() != null) { builder.setCompactionReason(compactionReason); } From 1a774a5748ac995691e0b48e591ab52ecf1e5c38 Mon Sep 17 00:00:00 2001 From: saketa Date: Mon, 12 May 2025 15:00:29 -0700 Subject: [PATCH 17/18] HDDS-12501. Fixed checkstyle and build issue. --- .../compaction/log/CompactionLogEntry.java | 6 +- hadoop-ozone/integration-test/audit.log | 58 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/integration-test/audit.log diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java index 83b456e1e767..85759e429210 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java @@ -132,9 +132,9 @@ public String toString() { public Builder toBuilder() { Builder builder = new Builder(this.getDbSequenceNumber(), this.getCompactionTime(), this.getInputFileInfoList(), this.getOutputFileInfoList()); - String compactionReason = this.getCompactionReason(); - if (compactionLogEntry.getCompactionReason() != null) { - builder.setCompactionReason(compactionReason); + String reason = this.getCompactionReason(); + if (this.getCompactionReason() != null) { + builder.setCompactionReason(reason); } return builder; } diff --git a/hadoop-ozone/integration-test/audit.log b/hadoop-ozone/integration-test/audit.log new file mode 100644 index 000000000000..4ce6a6831abf --- /dev/null +++ b/hadoop-ozone/integration-test/audit.log @@ -0,0 +1,58 @@ +2025-05-12 14:54:53,129 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=GET_SERVICE_LIST {} | ret=SUCCESS | +2025-05-12 14:54:53,176 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_VOLUME {admin=schalamchala, owner=schalamchala, volume=a0b1d281-e390-433b-b950-59f1f1c42325, creationTime=1747086893160, modificationTime=1747086893160, quotaInBytes=-1, quotaInNamespace=-1, usedNamespace=0, objectID=-9223372036854775552, updateID=1, Transaction=1} | ret=SUCCESS | +2025-05-12 14:54:53,185 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_VOLUME {volume=a0b1d281-e390-433b-b950-59f1f1c42325} | ret=SUCCESS | +2025-05-12 14:54:53,194 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=71f8539d-236c-40e7-a915-a7d87e749381, bucketLayout=FILE_SYSTEM_OPTIMIZED, gdprEnabled=null, acls=[user:schalamchala:a[ACCESS], group:staff:rl[ACCESS]], isVersionEnabled=false, storageType=DISK, creationTime=1747086893192, bucketEncryptionKey=null, modificationTime=1747086893192, usedBytes=0, usedNamespace=0, owner=schalamchala, replicationType=null, replicationConfig=null, quotaInBytes=-1, quotaInNamespace=-1, Transaction=3} | ret=SUCCESS | +2025-05-12 14:54:53,214 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=71f8539d-236c-40e7-a915-a7d87e749381} | ret=SUCCESS | +2025-05-12 14:54:53,409 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=DELETE_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=71f8539d-236c-40e7-a915-a7d87e749381, Transaction=5} | ret=SUCCESS | +2025-05-12 14:54:53,436 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, bucketLayout=FILE_SYSTEM_OPTIMIZED, gdprEnabled=null, acls=[user:schalamchala:a[ACCESS], group:staff:rl[ACCESS]], isVersionEnabled=false, storageType=DISK, creationTime=1747086893434, bucketEncryptionKey=null, modificationTime=1747086893434, usedBytes=0, usedNamespace=0, owner=schalamchala, replicationType=null, replicationConfig=null, quotaInBytes=-1, quotaInNamespace=-1, Transaction=7} | ret=SUCCESS | +2025-05-12 14:54:53,438 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,441 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,462 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-0, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=9} | ret=SUCCESS | +2025-05-12 14:54:53,470 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,475 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-1, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=11} | ret=SUCCESS | +2025-05-12 14:54:53,476 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,480 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-2, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=13} | ret=SUCCESS | +2025-05-12 14:54:53,482 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,486 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-3, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=15} | ret=SUCCESS | +2025-05-12 14:54:53,487 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,490 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-4, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=17} | ret=SUCCESS | +2025-05-12 14:54:53,492 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,497 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-5, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=19} | ret=SUCCESS | +2025-05-12 14:54:53,502 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,511 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-6, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=21} | ret=SUCCESS | +2025-05-12 14:54:53,517 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,521 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-7, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=23} | ret=SUCCESS | +2025-05-12 14:54:53,523 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,525 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-8, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=25} | ret=SUCCESS | +2025-05-12 14:54:53,527 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | +2025-05-12 14:54:53,531 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-9, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=27} | ret=SUCCESS | +2025-05-12 14:58:23,646 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=GET_SERVICE_LIST {} | ret=SUCCESS | +2025-05-12 14:58:23,697 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_VOLUME {admin=schalamchala, owner=schalamchala, volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, creationTime=1747087103677, modificationTime=1747087103677, quotaInBytes=-1, quotaInNamespace=-1, usedNamespace=0, objectID=-9223372036854775552, updateID=1, Transaction=1} | ret=SUCCESS | +2025-05-12 14:58:23,707 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_VOLUME {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7} | ret=SUCCESS | +2025-05-12 14:58:23,716 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=438c9fc9-548e-415a-a191-2717d09493e5, bucketLayout=FILE_SYSTEM_OPTIMIZED, gdprEnabled=null, acls=[user:schalamchala:a[ACCESS], group:staff:rl[ACCESS]], isVersionEnabled=false, storageType=DISK, creationTime=1747087103713, bucketEncryptionKey=null, modificationTime=1747087103713, usedBytes=0, usedNamespace=0, owner=schalamchala, replicationType=null, replicationConfig=null, quotaInBytes=-1, quotaInNamespace=-1, Transaction=3} | ret=SUCCESS | +2025-05-12 14:58:23,894 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=DELETE_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=438c9fc9-548e-415a-a191-2717d09493e5, Transaction=5} | ret=SUCCESS | +2025-05-12 14:58:23,895 | ERROR | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=438c9fc9-548e-415a-a191-2717d09493e5} | ret=FAILURE | BUCKET_NOT_FOUND org.apache.hadoop.ozone.om.exceptions.OMException: Bucket not found: 72e0edca-ba7f-448c-bbd0-d172fd9823d7/438c9fc9-548e-415a-a191-2717d09493e5 + at org.apache.hadoop.ozone.om.OzoneManagerUtils.reportNotFound(OzoneManagerUtils.java:85) + at org.apache.hadoop.ozone.om.OzoneManagerUtils.getBucketInfo(OzoneManagerUtils.java:70) +2025-05-12 14:58:23,901 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, bucketLayout=FILE_SYSTEM_OPTIMIZED, gdprEnabled=null, acls=[user:schalamchala:a[ACCESS], group:staff:rl[ACCESS]], isVersionEnabled=false, storageType=DISK, creationTime=1747087103900, bucketEncryptionKey=null, modificationTime=1747087103900, usedBytes=0, usedNamespace=0, owner=schalamchala, replicationType=null, replicationConfig=null, quotaInBytes=-1, quotaInNamespace=-1, Transaction=7} | ret=SUCCESS | +2025-05-12 14:58:23,904 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:23,906 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:23,972 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-0, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=9} | ret=SUCCESS | +2025-05-12 14:58:23,985 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:23,997 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-1, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=11} | ret=SUCCESS | +2025-05-12 14:58:23,999 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:24,003 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-2, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=13} | ret=SUCCESS | +2025-05-12 14:58:24,005 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:24,009 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-3, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=15} | ret=SUCCESS | +2025-05-12 14:58:24,010 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:24,014 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-4, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=17} | ret=SUCCESS | +2025-05-12 14:58:24,016 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:24,022 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-5, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=19} | ret=SUCCESS | +2025-05-12 14:58:24,023 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:24,027 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-6, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=21} | ret=SUCCESS | +2025-05-12 14:58:24,029 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:24,033 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-7, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=23} | ret=SUCCESS | +2025-05-12 14:58:24,035 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:24,039 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-8, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=25} | ret=SUCCESS | +2025-05-12 14:58:24,041 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | +2025-05-12 14:58:24,044 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-9, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=27} | ret=SUCCESS | From 6aa7525c4b31f2489be2c2272ca41e446dd14b9a Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Mon, 12 May 2025 15:29:01 -0700 Subject: [PATCH 18/18] Delete hadoop-ozone/integration-test/audit.log --- hadoop-ozone/integration-test/audit.log | 58 ------------------------- 1 file changed, 58 deletions(-) delete mode 100644 hadoop-ozone/integration-test/audit.log diff --git a/hadoop-ozone/integration-test/audit.log b/hadoop-ozone/integration-test/audit.log deleted file mode 100644 index 4ce6a6831abf..000000000000 --- a/hadoop-ozone/integration-test/audit.log +++ /dev/null @@ -1,58 +0,0 @@ -2025-05-12 14:54:53,129 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=GET_SERVICE_LIST {} | ret=SUCCESS | -2025-05-12 14:54:53,176 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_VOLUME {admin=schalamchala, owner=schalamchala, volume=a0b1d281-e390-433b-b950-59f1f1c42325, creationTime=1747086893160, modificationTime=1747086893160, quotaInBytes=-1, quotaInNamespace=-1, usedNamespace=0, objectID=-9223372036854775552, updateID=1, Transaction=1} | ret=SUCCESS | -2025-05-12 14:54:53,185 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_VOLUME {volume=a0b1d281-e390-433b-b950-59f1f1c42325} | ret=SUCCESS | -2025-05-12 14:54:53,194 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=71f8539d-236c-40e7-a915-a7d87e749381, bucketLayout=FILE_SYSTEM_OPTIMIZED, gdprEnabled=null, acls=[user:schalamchala:a[ACCESS], group:staff:rl[ACCESS]], isVersionEnabled=false, storageType=DISK, creationTime=1747086893192, bucketEncryptionKey=null, modificationTime=1747086893192, usedBytes=0, usedNamespace=0, owner=schalamchala, replicationType=null, replicationConfig=null, quotaInBytes=-1, quotaInNamespace=-1, Transaction=3} | ret=SUCCESS | -2025-05-12 14:54:53,214 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=71f8539d-236c-40e7-a915-a7d87e749381} | ret=SUCCESS | -2025-05-12 14:54:53,409 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=DELETE_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=71f8539d-236c-40e7-a915-a7d87e749381, Transaction=5} | ret=SUCCESS | -2025-05-12 14:54:53,436 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, bucketLayout=FILE_SYSTEM_OPTIMIZED, gdprEnabled=null, acls=[user:schalamchala:a[ACCESS], group:staff:rl[ACCESS]], isVersionEnabled=false, storageType=DISK, creationTime=1747086893434, bucketEncryptionKey=null, modificationTime=1747086893434, usedBytes=0, usedNamespace=0, owner=schalamchala, replicationType=null, replicationConfig=null, quotaInBytes=-1, quotaInNamespace=-1, Transaction=7} | ret=SUCCESS | -2025-05-12 14:54:53,438 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,441 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,462 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-0, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=9} | ret=SUCCESS | -2025-05-12 14:54:53,470 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,475 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-1, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=11} | ret=SUCCESS | -2025-05-12 14:54:53,476 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,480 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-2, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=13} | ret=SUCCESS | -2025-05-12 14:54:53,482 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,486 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-3, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=15} | ret=SUCCESS | -2025-05-12 14:54:53,487 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,490 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-4, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=17} | ret=SUCCESS | -2025-05-12 14:54:53,492 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,497 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-5, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=19} | ret=SUCCESS | -2025-05-12 14:54:53,502 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,511 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-6, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=21} | ret=SUCCESS | -2025-05-12 14:54:53,517 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,521 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-7, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=23} | ret=SUCCESS | -2025-05-12 14:54:53,523 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,525 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-8, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=25} | ret=SUCCESS | -2025-05-12 14:54:53,527 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20} | ret=SUCCESS | -2025-05-12 14:54:53,531 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=a0b1d281-e390-433b-b950-59f1f1c42325, bucket=015b331a-36e5-45fd-97bc-f4fe12b62b20, key=key-9, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=27} | ret=SUCCESS | -2025-05-12 14:58:23,646 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=GET_SERVICE_LIST {} | ret=SUCCESS | -2025-05-12 14:58:23,697 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_VOLUME {admin=schalamchala, owner=schalamchala, volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, creationTime=1747087103677, modificationTime=1747087103677, quotaInBytes=-1, quotaInNamespace=-1, usedNamespace=0, objectID=-9223372036854775552, updateID=1, Transaction=1} | ret=SUCCESS | -2025-05-12 14:58:23,707 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_VOLUME {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7} | ret=SUCCESS | -2025-05-12 14:58:23,716 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=438c9fc9-548e-415a-a191-2717d09493e5, bucketLayout=FILE_SYSTEM_OPTIMIZED, gdprEnabled=null, acls=[user:schalamchala:a[ACCESS], group:staff:rl[ACCESS]], isVersionEnabled=false, storageType=DISK, creationTime=1747087103713, bucketEncryptionKey=null, modificationTime=1747087103713, usedBytes=0, usedNamespace=0, owner=schalamchala, replicationType=null, replicationConfig=null, quotaInBytes=-1, quotaInNamespace=-1, Transaction=3} | ret=SUCCESS | -2025-05-12 14:58:23,894 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=DELETE_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=438c9fc9-548e-415a-a191-2717d09493e5, Transaction=5} | ret=SUCCESS | -2025-05-12 14:58:23,895 | ERROR | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=438c9fc9-548e-415a-a191-2717d09493e5} | ret=FAILURE | BUCKET_NOT_FOUND org.apache.hadoop.ozone.om.exceptions.OMException: Bucket not found: 72e0edca-ba7f-448c-bbd0-d172fd9823d7/438c9fc9-548e-415a-a191-2717d09493e5 - at org.apache.hadoop.ozone.om.OzoneManagerUtils.reportNotFound(OzoneManagerUtils.java:85) - at org.apache.hadoop.ozone.om.OzoneManagerUtils.getBucketInfo(OzoneManagerUtils.java:70) -2025-05-12 14:58:23,901 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=CREATE_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, bucketLayout=FILE_SYSTEM_OPTIMIZED, gdprEnabled=null, acls=[user:schalamchala:a[ACCESS], group:staff:rl[ACCESS]], isVersionEnabled=false, storageType=DISK, creationTime=1747087103900, bucketEncryptionKey=null, modificationTime=1747087103900, usedBytes=0, usedNamespace=0, owner=schalamchala, replicationType=null, replicationConfig=null, quotaInBytes=-1, quotaInNamespace=-1, Transaction=7} | ret=SUCCESS | -2025-05-12 14:58:23,904 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:23,906 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:23,972 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-0, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=9} | ret=SUCCESS | -2025-05-12 14:58:23,985 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:23,997 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-1, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=11} | ret=SUCCESS | -2025-05-12 14:58:23,999 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:24,003 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-2, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=13} | ret=SUCCESS | -2025-05-12 14:58:24,005 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:24,009 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-3, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=15} | ret=SUCCESS | -2025-05-12 14:58:24,010 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:24,014 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-4, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=17} | ret=SUCCESS | -2025-05-12 14:58:24,016 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:24,022 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-5, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=19} | ret=SUCCESS | -2025-05-12 14:58:24,023 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:24,027 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-6, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=21} | ret=SUCCESS | -2025-05-12 14:58:24,029 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:24,033 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-7, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=23} | ret=SUCCESS | -2025-05-12 14:58:24,035 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:24,039 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-8, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=25} | ret=SUCCESS | -2025-05-12 14:58:24,041 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=READ_BUCKET {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513} | ret=SUCCESS | -2025-05-12 14:58:24,044 | INFO | OMAudit | user=schalamchala | ip=127.0.0.1 | op=ALLOCATE_KEY {volume=72e0edca-ba7f-448c-bbd0-d172fd9823d7, bucket=d9b0bdbe-d44c-4466-8579-d591372b8513, key=key-9, dataSize=12, replicationType=RATIS, replicationFactor=ONE, Transaction=27} | ret=SUCCESS |