From c7dba882e5fad2bb730ad5653ace7e91efb08ee5 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 28 Jul 2022 20:35:23 +0800 Subject: [PATCH 01/10] tuning allocator pooling concurrency. --- .../bookie/storage/ldb/DbLedgerStorage.java | 4 +- .../bookkeeper/conf/ServerConfiguration.java | 47 ++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java index ab5bb0e172f..d9f422c6f80 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java @@ -98,9 +98,9 @@ public class DbLedgerStorage implements LedgerStorage { private static final int MB = 1024 * 1024; - private static final long DEFAULT_WRITE_CACHE_MAX_SIZE_MB = + public static final long DEFAULT_WRITE_CACHE_MAX_SIZE_MB = (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; - private static final long DEFAULT_READ_CACHE_MAX_SIZE_MB = + public static final long DEFAULT_READ_CACHE_MAX_SIZE_MB = (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; static final String READ_AHEAD_CACHE_BATCH_SIZE = "dbStorage_readAheadCacheBatchSize"; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 31177b011e4..cca970c782a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -23,6 +23,8 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; // CHECKSTYLE.OFF: IllegalImport +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.util.NettyRuntime; import io.netty.util.internal.PlatformDependent; // CHECKSTYLE.ON: IllegalImport import java.io.File; @@ -3857,7 +3859,50 @@ public ServerConfiguration setEntryLogPerLedgerCounterLimitsMultFactor( public boolean isLocalConsistencyCheckOnStartup() { return this.getBoolean(LOCAL_CONSISTENCY_CHECK_ON_STARTUP, false); } - + + /** + * The configured pooling concurrency for the allocator, if user config it, we should consider the unpooled + * direct memory which readCache and writeCache occupy when use DbLedgerStorage. + */ + public int getAllocatorPoolingConcurrency() { + Integer allocatorPoolingConcurrency = this.getInteger(ALLOCATOR_POOLING_CONCURRENCY, null); + if (allocatorPoolingConcurrency != null) { + return allocatorPoolingConcurrency; + } + String ledgerStorageClass = getLedgerStorageClass(); + if (DbLedgerStorage.class.getName().equals(ledgerStorageClass)) { + long writeCacheMb; + Object writeCacheConf = this.getProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB); + if (writeCacheConf instanceof Number) { + writeCacheMb = ((Number) writeCacheConf).longValue(); + } else if (writeCacheConf == null) { + writeCacheMb = DbLedgerStorage.DEFAULT_WRITE_CACHE_MAX_SIZE_MB; + } else if (StringUtils.isEmpty(this.getString(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB))) { + writeCacheMb = DbLedgerStorage.DEFAULT_WRITE_CACHE_MAX_SIZE_MB; + } else { + writeCacheMb = this.getLong(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB); + } + long readCacheMb; + Object readCacheConf = this.getProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB); + if (readCacheConf instanceof Number) { + readCacheMb = ((Number) readCacheConf).longValue(); + } else if (readCacheConf == null) { + readCacheMb = DbLedgerStorage.DEFAULT_READ_CACHE_MAX_SIZE_MB; + } else if (StringUtils.isEmpty(this.getString(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB))) { + readCacheMb = DbLedgerStorage.DEFAULT_READ_CACHE_MAX_SIZE_MB; + } else { + readCacheMb = this.getLong(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB); + } + int mb = 1024 * 1024; + long availableDirectMemory = PlatformDependent.maxDirectMemory() - writeCacheMb * mb - readCacheMb * mb; + int defaultMinNumArena = NettyRuntime.availableProcessors() * 2; + final int defaultChunkSize = PooledByteBufAllocator.defaultPageSize() << PooledByteBufAllocator.defaultMaxOrder(); + int suitableNum = (int) (availableDirectMemory / defaultChunkSize / 2 / 3); + return Math.min(defaultMinNumArena, suitableNum); + } + return super.getAllocatorPoolingConcurrency(); + } + /** * Get the authorized roles. * From 5e9445396a1e1b11206d2ab3c0ee4a8a4cbced98 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 28 Jul 2022 20:36:04 +0800 Subject: [PATCH 02/10] fix code style. --- .../java/org/apache/bookkeeper/conf/ServerConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index cca970c782a..26f7211641d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -3859,7 +3859,7 @@ public ServerConfiguration setEntryLogPerLedgerCounterLimitsMultFactor( public boolean isLocalConsistencyCheckOnStartup() { return this.getBoolean(LOCAL_CONSISTENCY_CHECK_ON_STARTUP, false); } - + /** * The configured pooling concurrency for the allocator, if user config it, we should consider the unpooled * direct memory which readCache and writeCache occupy when use DbLedgerStorage. @@ -3902,7 +3902,7 @@ public int getAllocatorPoolingConcurrency() { } return super.getAllocatorPoolingConcurrency(); } - + /** * Get the authorized roles. * From ea0c5d8e7ff36fa5a9bad806283b260f65eda8b4 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 28 Jul 2022 21:21:18 +0800 Subject: [PATCH 03/10] fix code style. --- .../java/org/apache/bookkeeper/conf/ServerConfiguration.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 26f7211641d..e0e66aec3b4 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -3896,7 +3896,8 @@ public int getAllocatorPoolingConcurrency() { int mb = 1024 * 1024; long availableDirectMemory = PlatformDependent.maxDirectMemory() - writeCacheMb * mb - readCacheMb * mb; int defaultMinNumArena = NettyRuntime.availableProcessors() * 2; - final int defaultChunkSize = PooledByteBufAllocator.defaultPageSize() << PooledByteBufAllocator.defaultMaxOrder(); + final int defaultChunkSize = + PooledByteBufAllocator.defaultPageSize() << PooledByteBufAllocator.defaultMaxOrder(); int suitableNum = (int) (availableDirectMemory / defaultChunkSize / 2 / 3); return Math.min(defaultMinNumArena, suitableNum); } From 45098f66c894fb3a391fa5d2bcf7da2700d977b5 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Fri, 29 Jul 2022 18:56:05 +0800 Subject: [PATCH 04/10] Use DbLedgerStorageConfiguration to config DbLedgerStorage --- .../bookie/storage/ldb/DbLedgerStorage.java | 79 +++-------- .../ldb/SingleDirectoryDbLedgerStorage.java | 6 +- .../conf/DbLedgerStorageConfiguration.java | 132 ++++++++++++++++++ .../bookkeeper/conf/ServerConfiguration.java | 44 ------ .../org/apache/bookkeeper/server/Main.java | 14 +- 5 files changed, 168 insertions(+), 107 deletions(-) create mode 100644 bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java index d9f422c6f80..cbf5987e8d3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java @@ -65,6 +65,7 @@ import org.apache.bookkeeper.common.util.MathUtils; import org.apache.bookkeeper.common.util.Watcher; import org.apache.bookkeeper.common.util.nativeio.NativeIOImpl; +import org.apache.bookkeeper.conf.DbLedgerStorageConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.slogger.slf4j.Slf4jSlogger; @@ -82,40 +83,6 @@ @Slf4j public class DbLedgerStorage implements LedgerStorage { - public static final String WRITE_CACHE_MAX_SIZE_MB = "dbStorage_writeCacheMaxSizeMb"; - public static final String READ_AHEAD_CACHE_MAX_SIZE_MB = "dbStorage_readAheadCacheMaxSizeMb"; - public static final String DIRECT_IO_ENTRYLOGGER = "dbStorage_directIOEntryLogger"; - public static final String DIRECT_IO_ENTRYLOGGER_TOTAL_WRITEBUFFER_SIZE_MB = - "dbStorage_directIOEntryLoggerTotalWriteBufferSizeMb"; - public static final String DIRECT_IO_ENTRYLOGGER_TOTAL_READBUFFER_SIZE_MB = - "dbStorage_directIOEntryLoggerTotalReadBufferSizeMb"; - public static final String DIRECT_IO_ENTRYLOGGER_READBUFFER_SIZE_MB = - "dbStorage_directIOEntryLoggerReadBufferSizeMb"; - public static final String DIRECT_IO_ENTRYLOGGER_MAX_FD_CACHE_TIME_SECONDS = - "dbStorage_directIOEntryLoggerMaxFdCacheTimeSeconds"; - - static final String MAX_THROTTLE_TIME_MILLIS = "dbStorage_maxThrottleTimeMs"; - - private static final int MB = 1024 * 1024; - - public static final long DEFAULT_WRITE_CACHE_MAX_SIZE_MB = - (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; - public static final long DEFAULT_READ_CACHE_MAX_SIZE_MB = - (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; - - static final String READ_AHEAD_CACHE_BATCH_SIZE = "dbStorage_readAheadCacheBatchSize"; - private static final int DEFAULT_READ_AHEAD_CACHE_BATCH_SIZE = 100; - - private static final long DEFAULT_DIRECT_IO_TOTAL_WRITEBUFFER_SIZE_MB = - (long) (0.125 * PlatformDependent.estimateMaxDirectMemory()) - / MB; - private static final long DEFAULT_DIRECT_IO_TOTAL_READBUFFER_SIZE_MB = - (long) (0.125 * PlatformDependent.estimateMaxDirectMemory()) - / MB; - private static final long DEFAULT_DIRECT_IO_READBUFFER_SIZE_MB = 8; - - private static final int DEFAULT_DIRECT_IO_MAX_FD_CACHE_TIME_SECONDS = 300; - // use the storage assigned to ledger 0 for flags. // if the storage configuration changes, the flags may be lost // but in that case data integrity should kick off anyhow. @@ -150,19 +117,18 @@ public class DbLedgerStorage implements LedgerStorage { public void initialize(ServerConfiguration conf, LedgerManager ledgerManager, LedgerDirsManager ledgerDirsManager, LedgerDirsManager indexDirsManager, StatsLogger statsLogger, ByteBufAllocator allocator) throws IOException { - long writeCacheMaxSize = getLongVariableOrDefault(conf, WRITE_CACHE_MAX_SIZE_MB, - DEFAULT_WRITE_CACHE_MAX_SIZE_MB) * MB; - long readCacheMaxSize = getLongVariableOrDefault(conf, READ_AHEAD_CACHE_MAX_SIZE_MB, - DEFAULT_READ_CACHE_MAX_SIZE_MB) * MB; - boolean directIOEntryLogger = getBooleanVariableOrDefault(conf, DIRECT_IO_ENTRYLOGGER, false); + + long writeCacheMaxSize = ((DbLedgerStorageConfiguration) conf).getWriteCacheMaxSize(); + long readCacheMaxSize = ((DbLedgerStorageConfiguration) conf).getReadCacheMaxSize(); + boolean directIOEntryLogger = ((DbLedgerStorageConfiguration) conf).isDirectIOEntryLoggerEnabled(); this.allocator = allocator; this.numberOfDirs = ledgerDirsManager.getAllLedgerDirs().size(); log.info("Started Db Ledger Storage"); log.info(" - Number of directories: {}", numberOfDirs); - log.info(" - Write cache size: {} MB", writeCacheMaxSize / MB); - log.info(" - Read Cache: {} MB", readCacheMaxSize / MB); + log.info(" - Write cache size: {} MB", writeCacheMaxSize / 1024 / 1024); + log.info(" - Read Cache: {} MB", readCacheMaxSize / 1024 / 1024); if (readCacheMaxSize + writeCacheMaxSize > PlatformDependent.estimateMaxDirectMemory()) { throw new IOException("Read and write cache sizes exceed the configured max direct memory size"); @@ -174,7 +140,7 @@ public void initialize(ServerConfiguration conf, LedgerManager ledgerManager, Le long perDirectoryWriteCacheSize = writeCacheMaxSize / numberOfDirs; long perDirectoryReadCacheSize = readCacheMaxSize / numberOfDirs; - int readAheadCacheBatchSize = conf.getInt(READ_AHEAD_CACHE_BATCH_SIZE, DEFAULT_READ_AHEAD_CACHE_BATCH_SIZE); + int readAheadCacheBatchSize = ((DbLedgerStorageConfiguration) conf).getReadAheadCacheBatchSize(); gcExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollector")); @@ -198,22 +164,19 @@ public void initialize(ServerConfiguration conf, LedgerManager ledgerManager, Le EntryLogger entrylogger; if (directIOEntryLogger) { - long perDirectoryTotalWriteBufferSize = MB * getLongVariableOrDefault( - conf, - DIRECT_IO_ENTRYLOGGER_TOTAL_WRITEBUFFER_SIZE_MB, - DEFAULT_DIRECT_IO_TOTAL_WRITEBUFFER_SIZE_MB) / numberOfDirs; - long perDirectoryTotalReadBufferSize = MB * getLongVariableOrDefault( - conf, - DIRECT_IO_ENTRYLOGGER_TOTAL_READBUFFER_SIZE_MB, - DEFAULT_DIRECT_IO_TOTAL_READBUFFER_SIZE_MB) / numberOfDirs; - int readBufferSize = MB * (int) getLongVariableOrDefault( - conf, - DIRECT_IO_ENTRYLOGGER_READBUFFER_SIZE_MB, - DEFAULT_DIRECT_IO_READBUFFER_SIZE_MB); - int maxFdCacheTimeSeconds = (int) getLongVariableOrDefault( - conf, - DIRECT_IO_ENTRYLOGGER_MAX_FD_CACHE_TIME_SECONDS, - DEFAULT_DIRECT_IO_MAX_FD_CACHE_TIME_SECONDS); + long perDirectoryTotalWriteBufferSize = + ((DbLedgerStorageConfiguration) conf).getDirectIOEntryLoggerTotalWriteBufferSize() + / numberOfDirs; + long perDirectoryTotalReadBufferSize = + ((DbLedgerStorageConfiguration) conf).getDirectIOEntryLoggerTotalReadBufferSize() + / numberOfDirs; + + int readBufferSize = (int) ((DbLedgerStorageConfiguration) conf) + .getDirectIOEntryLoggerReadBufferSize(); + + int maxFdCacheTimeSeconds = (int) ((DbLedgerStorageConfiguration) conf) + .getDirectIOEntryLoggerMaxFDCacheTimeSeconds(); + Slf4jSlogger slog = new Slf4jSlogger(DbLedgerStorage.class); entryLoggerWriteExecutor = Executors.newSingleThreadExecutor( new DefaultThreadFactory("EntryLoggerWrite")); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index a1131df8157..617e4c0ad2a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -71,6 +71,7 @@ import org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorageDataFormats.LedgerData; import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorage.Batch; import org.apache.bookkeeper.common.util.Watcher; +import org.apache.bookkeeper.conf.DbLedgerStorageConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.proto.BookieProtocol; @@ -139,8 +140,6 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage private final DbLedgerStorageStats dbLedgerStorageStats; - private static final long DEFAULT_MAX_THROTTLE_TIME_MILLIS = TimeUnit.SECONDS.toMillis(10); - private final long maxReadAheadBytesSize; private final Counter flushExecutorTime; @@ -178,8 +177,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le // Do not attempt to perform read-ahead more than half the total size of the cache maxReadAheadBytesSize = readCacheMaxSize / 2; - long maxThrottleTimeMillis = conf.getLong(DbLedgerStorage.MAX_THROTTLE_TIME_MILLIS, - DEFAULT_MAX_THROTTLE_TIME_MILLIS); + long maxThrottleTimeMillis = ((DbLedgerStorageConfiguration) conf).getMaxThrottleTimeMillis(); maxThrottleTimeNanos = TimeUnit.MILLISECONDS.toNanos(maxThrottleTimeMillis); readCache = new ReadCache(allocator, readCacheMaxSize); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java new file mode 100644 index 00000000000..14f2beddc26 --- /dev/null +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java @@ -0,0 +1,132 @@ +package org.apache.bookkeeper.conf; + +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.util.NettyRuntime; +import io.netty.util.internal.PlatformDependent; +import org.apache.commons.lang3.StringUtils; + +import java.util.concurrent.TimeUnit; + +public class DbLedgerStorageConfiguration extends ServerConfiguration { + + private static final int MB = 1024 * 1024; + + private static final String WRITE_CACHE_MAX_SIZE_MB = "dbStorage_writeCacheMaxSizeMb"; + + private static final long DEFAULT_WRITE_CACHE_MAX_SIZE_MB = + (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; + + private static final String READ_AHEAD_CACHE_MAX_SIZE_MB = "dbStorage_readAheadCacheMaxSizeMb"; + + private static final long DEFAULT_READ_CACHE_MAX_SIZE_MB = + (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; + + private static final String READ_AHEAD_CACHE_BATCH_SIZE = "dbStorage_readAheadCacheBatchSize"; + + private static final int DEFAULT_READ_AHEAD_CACHE_BATCH_SIZE = 100; + + private static final String DIRECT_IO_ENTRYLOGGER = "dbStorage_directIOEntryLogger"; + + private static final String DIRECT_IO_ENTRYLOGGER_TOTAL_WRITEBUFFER_SIZE_MB = "dbStorage_directIOEntryLoggerTotalWriteBufferSizeMb"; + + private static final long DEFAULT_DIRECT_IO_TOTAL_WRITEBUFFER_SIZE_MB = + (long) (0.125 * PlatformDependent.estimateMaxDirectMemory()) / MB; + + private static final String DIRECT_IO_ENTRYLOGGER_TOTAL_READBUFFER_SIZE_MB = "dbStorage_directIOEntryLoggerTotalReadBufferSizeMb"; + + private static final long DEFAULT_DIRECT_IO_TOTAL_READBUFFER_SIZE_MB = + (long) (0.125 * PlatformDependent.estimateMaxDirectMemory()) / MB; + + private static final String DIRECT_IO_ENTRYLOGGER_READBUFFER_SIZE_MB = "dbStorage_directIOEntryLoggerReadBufferSizeMb"; + + private static final long DEFAULT_DIRECT_IO_READBUFFER_SIZE_MB = 8; + + private static final String DIRECT_IO_ENTRYLOGGER_MAX_FD_CACHE_TIME_SECONDS = "dbStorage_directIOEntryLoggerMaxFdCacheTimeSeconds"; + + private static final int DEFAULT_DIRECT_IO_MAX_FD_CACHE_TIME_SECONDS = 300; + + private static final String MAX_THROTTLE_TIME_MILLIS = "dbStorage_maxThrottleTimeMs"; + + private static final long DEFAULT_MAX_THROTTLE_TIME_MILLIS = TimeUnit.SECONDS.toMillis(10); + + public long getWriteCacheMaxSize() { + return getLongVariableOrDefault(WRITE_CACHE_MAX_SIZE_MB, DEFAULT_WRITE_CACHE_MAX_SIZE_MB) * MB; + } + + public long getReadCacheMaxSize() { + return getLongVariableOrDefault(READ_AHEAD_CACHE_MAX_SIZE_MB, DEFAULT_READ_CACHE_MAX_SIZE_MB) * MB; + } + + public int getReadAheadCacheBatchSize() { + return this.getInt(READ_AHEAD_CACHE_BATCH_SIZE, DEFAULT_READ_AHEAD_CACHE_BATCH_SIZE); + } + + public boolean isDirectIOEntryLoggerEnabled() { + return getBooleanVariableOrDefault(DIRECT_IO_ENTRYLOGGER, false); + } + + public long getDirectIOEntryLoggerTotalWriteBufferSize() { + return getLongVariableOrDefault(DIRECT_IO_ENTRYLOGGER_TOTAL_WRITEBUFFER_SIZE_MB, + DEFAULT_DIRECT_IO_TOTAL_WRITEBUFFER_SIZE_MB) * MB; + } + + public long getDirectIOEntryLoggerTotalReadBufferSize() { + return getLongVariableOrDefault(DIRECT_IO_ENTRYLOGGER_TOTAL_READBUFFER_SIZE_MB, + DEFAULT_DIRECT_IO_TOTAL_READBUFFER_SIZE_MB) * MB; + } + + public long getDirectIOEntryLoggerReadBufferSize() { + return getLongVariableOrDefault(DIRECT_IO_ENTRYLOGGER_READBUFFER_SIZE_MB, DEFAULT_DIRECT_IO_READBUFFER_SIZE_MB) + * MB; + } + + public long getDirectIOEntryLoggerMaxFDCacheTimeSeconds() { + return getLongVariableOrDefault(DIRECT_IO_ENTRYLOGGER_MAX_FD_CACHE_TIME_SECONDS, + DEFAULT_DIRECT_IO_MAX_FD_CACHE_TIME_SECONDS); + } + + public long getMaxThrottleTimeMillis() { + return this.getLong(MAX_THROTTLE_TIME_MILLIS, DEFAULT_MAX_THROTTLE_TIME_MILLIS); + } + + /** + * The configured pooling concurrency for the allocator, if user config it, we should consider the unpooled direct + * memory which readCache and writeCache occupy when use DbLedgerStorage. + */ + public int getAllocatorPoolingConcurrency() { + long writeCacheSize = this.getWriteCacheMaxSize(); + long readCacheSize = this.getReadCacheMaxSize(); + long availableDirectMemory = PlatformDependent.maxDirectMemory() - writeCacheSize - readCacheSize; + int defaultMinNumArena = NettyRuntime.availableProcessors() * 2; + final int defaultChunkSize = + PooledByteBufAllocator.defaultPageSize() << PooledByteBufAllocator.defaultMaxOrder(); + int suitableNum = (int) (availableDirectMemory / defaultChunkSize / 2 / 3); + return Math.min(defaultMinNumArena, suitableNum); + } + + private long getLongVariableOrDefault(String keyName, long defaultValue) { + Object obj = this.getProperty(keyName); + if (obj instanceof Number) { + return ((Number) obj).longValue(); + } else if (obj == null) { + return defaultValue; + } else if (StringUtils.isEmpty(this.getString(keyName))) { + return defaultValue; + } else { + return this.getLong(keyName); + } + } + + private boolean getBooleanVariableOrDefault(String keyName, boolean defaultValue) { + Object obj = this.getProperty(keyName); + if (obj instanceof Boolean) { + return (Boolean) obj; + } else if (obj == null) { + return defaultValue; + } else if (StringUtils.isEmpty(this.getString(keyName))) { + return defaultValue; + } else { + return this.getBoolean(keyName); + } + } +} diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index e0e66aec3b4..1741b7bdd72 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -3860,50 +3860,6 @@ public boolean isLocalConsistencyCheckOnStartup() { return this.getBoolean(LOCAL_CONSISTENCY_CHECK_ON_STARTUP, false); } - /** - * The configured pooling concurrency for the allocator, if user config it, we should consider the unpooled - * direct memory which readCache and writeCache occupy when use DbLedgerStorage. - */ - public int getAllocatorPoolingConcurrency() { - Integer allocatorPoolingConcurrency = this.getInteger(ALLOCATOR_POOLING_CONCURRENCY, null); - if (allocatorPoolingConcurrency != null) { - return allocatorPoolingConcurrency; - } - String ledgerStorageClass = getLedgerStorageClass(); - if (DbLedgerStorage.class.getName().equals(ledgerStorageClass)) { - long writeCacheMb; - Object writeCacheConf = this.getProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB); - if (writeCacheConf instanceof Number) { - writeCacheMb = ((Number) writeCacheConf).longValue(); - } else if (writeCacheConf == null) { - writeCacheMb = DbLedgerStorage.DEFAULT_WRITE_CACHE_MAX_SIZE_MB; - } else if (StringUtils.isEmpty(this.getString(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB))) { - writeCacheMb = DbLedgerStorage.DEFAULT_WRITE_CACHE_MAX_SIZE_MB; - } else { - writeCacheMb = this.getLong(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB); - } - long readCacheMb; - Object readCacheConf = this.getProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB); - if (readCacheConf instanceof Number) { - readCacheMb = ((Number) readCacheConf).longValue(); - } else if (readCacheConf == null) { - readCacheMb = DbLedgerStorage.DEFAULT_READ_CACHE_MAX_SIZE_MB; - } else if (StringUtils.isEmpty(this.getString(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB))) { - readCacheMb = DbLedgerStorage.DEFAULT_READ_CACHE_MAX_SIZE_MB; - } else { - readCacheMb = this.getLong(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB); - } - int mb = 1024 * 1024; - long availableDirectMemory = PlatformDependent.maxDirectMemory() - writeCacheMb * mb - readCacheMb * mb; - int defaultMinNumArena = NettyRuntime.availableProcessors() * 2; - final int defaultChunkSize = - PooledByteBufAllocator.defaultPageSize() << PooledByteBufAllocator.defaultMaxOrder(); - int suitableNum = (int) (availableDirectMemory / defaultChunkSize / 2 / 3); - return Math.min(defaultMinNumArena, suitableNum); - } - return super.getAllocatorPoolingConcurrency(); - } - /** * Get the authorized roles. * diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java index 2d9c7ca276c..1bf201ac66c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java @@ -60,6 +60,7 @@ import org.apache.bookkeeper.bookie.datainteg.DataIntegrityService; import org.apache.bookkeeper.bookie.datainteg.EntryCopier; import org.apache.bookkeeper.bookie.datainteg.EntryCopierImpl; +import org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeperAdmin; import org.apache.bookkeeper.common.allocator.ByteBufAllocatorWithOomHandler; @@ -70,6 +71,7 @@ import org.apache.bookkeeper.common.component.LifecycleComponentStack; import org.apache.bookkeeper.common.component.RxSchedulerLifecycleComponent; import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.DbLedgerStorageConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.UncheckedConfigurationException; import org.apache.bookkeeper.discover.BookieServiceInfo; @@ -311,7 +313,17 @@ private static ServerConfiguration parseCommandLine(String[] args) Arrays.asList(conf.getIndexDirNames() != null ? conf.getIndexDirNames() : conf.getLedgerDirNames())); log.info(hello); - return conf; + return toDbLedgerStorageIfNecessary(conf); + } + + private static ServerConfiguration toDbLedgerStorageIfNecessary(ServerConfiguration serverConfiguration) { + String ledgerStorageClass = serverConfiguration.getLedgerStorageClass(); + if (DbLedgerStorage.class.getName().equals(ledgerStorageClass)) { + DbLedgerStorageConfiguration dbLedgerStorageConfiguration = new DbLedgerStorageConfiguration(); + dbLedgerStorageConfiguration.copy(serverConfiguration); + return dbLedgerStorageConfiguration; + } + return serverConfiguration; } /** From 3218e8c7340695e3f0428eb52b4cd36d163d2cf8 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Fri, 29 Jul 2022 20:27:32 +0800 Subject: [PATCH 05/10] compatible with old version. --- .../bookie/storage/ldb/DbLedgerStorage.java | 22 ++++++++----------- .../ldb/SingleDirectoryDbLedgerStorage.java | 2 +- .../conf/DbLedgerStorageConfiguration.java | 9 ++++---- .../bookkeeper/conf/ServerConfiguration.java | 8 ++++++- .../org/apache/bookkeeper/server/Main.java | 12 +--------- .../ldb/DbLedgerStorageBookieTest.java | 5 +++-- .../storage/ldb/DbLedgerStorageTest.java | 5 +++-- 7 files changed, 29 insertions(+), 34 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java index cbf5987e8d3..ddae2801b10 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java @@ -117,10 +117,10 @@ public class DbLedgerStorage implements LedgerStorage { public void initialize(ServerConfiguration conf, LedgerManager ledgerManager, LedgerDirsManager ledgerDirsManager, LedgerDirsManager indexDirsManager, StatsLogger statsLogger, ByteBufAllocator allocator) throws IOException { - - long writeCacheMaxSize = ((DbLedgerStorageConfiguration) conf).getWriteCacheMaxSize(); - long readCacheMaxSize = ((DbLedgerStorageConfiguration) conf).getReadCacheMaxSize(); - boolean directIOEntryLogger = ((DbLedgerStorageConfiguration) conf).isDirectIOEntryLoggerEnabled(); + DbLedgerStorageConfiguration dbLedgerStorageConf = conf.toDbLedgerStorageConfiguration(); + long writeCacheMaxSize = dbLedgerStorageConf.getWriteCacheMaxSize(); + long readCacheMaxSize = dbLedgerStorageConf.getReadCacheMaxSize(); + boolean directIOEntryLogger = dbLedgerStorageConf.isDirectIOEntryLoggerEnabled(); this.allocator = allocator; this.numberOfDirs = ledgerDirsManager.getAllLedgerDirs().size(); @@ -140,7 +140,7 @@ public void initialize(ServerConfiguration conf, LedgerManager ledgerManager, Le long perDirectoryWriteCacheSize = writeCacheMaxSize / numberOfDirs; long perDirectoryReadCacheSize = readCacheMaxSize / numberOfDirs; - int readAheadCacheBatchSize = ((DbLedgerStorageConfiguration) conf).getReadAheadCacheBatchSize(); + int readAheadCacheBatchSize = dbLedgerStorageConf.getReadAheadCacheBatchSize(); gcExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollector")); @@ -165,17 +165,13 @@ public void initialize(ServerConfiguration conf, LedgerManager ledgerManager, Le EntryLogger entrylogger; if (directIOEntryLogger) { long perDirectoryTotalWriteBufferSize = - ((DbLedgerStorageConfiguration) conf).getDirectIOEntryLoggerTotalWriteBufferSize() - / numberOfDirs; + dbLedgerStorageConf.getDirectIOEntryLoggerTotalWriteBufferSize() / numberOfDirs; long perDirectoryTotalReadBufferSize = - ((DbLedgerStorageConfiguration) conf).getDirectIOEntryLoggerTotalReadBufferSize() - / numberOfDirs; + dbLedgerStorageConf.getDirectIOEntryLoggerTotalReadBufferSize() / numberOfDirs; - int readBufferSize = (int) ((DbLedgerStorageConfiguration) conf) - .getDirectIOEntryLoggerReadBufferSize(); + int readBufferSize = (int) dbLedgerStorageConf.getDirectIOEntryLoggerReadBufferSize(); - int maxFdCacheTimeSeconds = (int) ((DbLedgerStorageConfiguration) conf) - .getDirectIOEntryLoggerMaxFDCacheTimeSeconds(); + int maxFdCacheTimeSeconds = (int) dbLedgerStorageConf.getDirectIOEntryLoggerMaxFDCacheTimeSeconds(); Slf4jSlogger slog = new Slf4jSlogger(DbLedgerStorage.class); entryLoggerWriteExecutor = Executors.newSingleThreadExecutor( diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 617e4c0ad2a..96a063d08bb 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -177,7 +177,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le // Do not attempt to perform read-ahead more than half the total size of the cache maxReadAheadBytesSize = readCacheMaxSize / 2; - long maxThrottleTimeMillis = ((DbLedgerStorageConfiguration) conf).getMaxThrottleTimeMillis(); + long maxThrottleTimeMillis = conf.toDbLedgerStorageConfiguration().getMaxThrottleTimeMillis(); maxThrottleTimeNanos = TimeUnit.MILLISECONDS.toNanos(maxThrottleTimeMillis); readCache = new ReadCache(allocator, readCacheMaxSize); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java index 14f2beddc26..9034c09fccf 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java @@ -1,5 +1,6 @@ package org.apache.bookkeeper.conf; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.NettyRuntime; import io.netty.util.internal.PlatformDependent; @@ -11,13 +12,13 @@ public class DbLedgerStorageConfiguration extends ServerConfiguration { private static final int MB = 1024 * 1024; - private static final String WRITE_CACHE_MAX_SIZE_MB = "dbStorage_writeCacheMaxSizeMb"; + public static final String WRITE_CACHE_MAX_SIZE_MB = "dbStorage_writeCacheMaxSizeMb"; private static final long DEFAULT_WRITE_CACHE_MAX_SIZE_MB = (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; - - private static final String READ_AHEAD_CACHE_MAX_SIZE_MB = "dbStorage_readAheadCacheMaxSizeMb"; - + + public static final String READ_AHEAD_CACHE_MAX_SIZE_MB = "dbStorage_readAheadCacheMaxSizeMb"; + private static final long DEFAULT_READ_CACHE_MAX_SIZE_MB = (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 1741b7bdd72..10c8eb32c84 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -336,7 +336,7 @@ public class ServerConfiguration extends AbstractConfiguration Date: Fri, 29 Jul 2022 20:30:20 +0800 Subject: [PATCH 06/10] add apache license. --- .../conf/DbLedgerStorageConfiguration.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java index 9034c09fccf..1da71206dd9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java @@ -1,6 +1,25 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ package org.apache.bookkeeper.conf; -import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.NettyRuntime; import io.netty.util.internal.PlatformDependent; From 75d4abcb15cf4a48c81202cadebf36fe3b1bcc1c Mon Sep 17 00:00:00 2001 From: horizonzy Date: Fri, 29 Jul 2022 20:45:10 +0800 Subject: [PATCH 07/10] override getAllocatorPoolingConcurrency --- .../conf/DbLedgerStorageConfiguration.java | 10 ++++++--- .../bookkeeper/conf/ServerConfiguration.java | 21 +++++++++++++++++++ .../ldb/DbLedgerStorageIndexDirTest.java | 5 +++-- .../ldb/DbLedgerStorageWriteCacheTest.java | 5 +++-- .../bookkeeper/conf/TestBKConfiguration.java | 4 ++-- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java index 1da71206dd9..3c478db2831 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java @@ -20,6 +20,7 @@ */ package org.apache.bookkeeper.conf; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.NettyRuntime; import io.netty.util.internal.PlatformDependent; @@ -31,11 +32,13 @@ public class DbLedgerStorageConfiguration extends ServerConfiguration { private static final int MB = 1024 * 1024; + @VisibleForTesting public static final String WRITE_CACHE_MAX_SIZE_MB = "dbStorage_writeCacheMaxSizeMb"; - + private static final long DEFAULT_WRITE_CACHE_MAX_SIZE_MB = (long) (0.25 * PlatformDependent.estimateMaxDirectMemory()) / MB; + @VisibleForTesting public static final String READ_AHEAD_CACHE_MAX_SIZE_MB = "dbStorage_readAheadCacheMaxSizeMb"; private static final long DEFAULT_READ_CACHE_MAX_SIZE_MB = @@ -64,8 +67,9 @@ public class DbLedgerStorageConfiguration extends ServerConfiguration { private static final String DIRECT_IO_ENTRYLOGGER_MAX_FD_CACHE_TIME_SECONDS = "dbStorage_directIOEntryLoggerMaxFdCacheTimeSeconds"; private static final int DEFAULT_DIRECT_IO_MAX_FD_CACHE_TIME_SECONDS = 300; - - private static final String MAX_THROTTLE_TIME_MILLIS = "dbStorage_maxThrottleTimeMs"; + + @VisibleForTesting + public static final String MAX_THROTTLE_TIME_MILLIS = "dbStorage_maxThrottleTimeMs"; private static final long DEFAULT_MAX_THROTTLE_TIME_MILLIS = TimeUnit.SECONDS.toMillis(10); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 10c8eb32c84..b5b01c35e26 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -4015,4 +4015,25 @@ public DbLedgerStorageConfiguration toDbLedgerStorageConfiguration() { dbLedgerStorageConfiguration.copy(this); return dbLedgerStorageConfiguration; } + + /** + * The configured pooling concurrency for the allocator, if user config it, we should consider the unpooled + * direct memory which readCache and writeCache occupy when use DbLedgerStorage. + */ + public int getAllocatorPoolingConcurrency() { + String ledgerStorageClass = getLedgerStorageClass(); + if (DbLedgerStorage.class.getName().equals(ledgerStorageClass)) { + DbLedgerStorageConfiguration dbLedgerStorageConfiguration = toDbLedgerStorageConfiguration(); + + long writeCacheSize = dbLedgerStorageConfiguration.getWriteCacheMaxSize(); + long readCacheSize = dbLedgerStorageConfiguration.getReadCacheMaxSize(); + long availableDirectMemory = PlatformDependent.maxDirectMemory() - writeCacheSize - readCacheSize; + + final int defaultChunkSize = PooledByteBufAllocator.defaultPageSize() << PooledByteBufAllocator.defaultMaxOrder(); + int defaultMinNumArena = NettyRuntime.availableProcessors() * 2; + return Math.min(defaultMinNumArena, (int) (availableDirectMemory / defaultChunkSize / 2 / 3)); + } + return super.getAllocatorPoolingConcurrency(); + } + } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java index 961f340c81c..c0cce41b861 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java @@ -34,6 +34,7 @@ import org.apache.bookkeeper.bookie.BookieException; import org.apache.bookkeeper.bookie.BookieImpl; import org.apache.bookkeeper.bookie.TestBookieImpl; +import org.apache.bookkeeper.conf.DbLedgerStorageConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; import org.apache.bookkeeper.proto.BookieProtocol; @@ -71,8 +72,8 @@ public void setup() throws Exception { conf.setGcWaitTime(gcWaitTime); /** the testcase cover specify indexDir for the class {@link SingleDirectoryDbLedgerStorage} */ conf.setLedgerStorageClass(DbLedgerStorage.class.getName()); - conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 1); - conf.setProperty(DbLedgerStorage.MAX_THROTTLE_TIME_MILLIS, 1000); + conf.setProperty(DbLedgerStorageConfiguration.WRITE_CACHE_MAX_SIZE_MB, 1); + conf.setProperty(DbLedgerStorageConfiguration.MAX_THROTTLE_TIME_MILLIS, 1000); conf.setLedgerDirNames(new String[]{tmpLedgerDir.toString()}); conf.setIndexDirName(new String[]{tmpIndexDir.toString()}); Bookie bookie = new TestBookieImpl(conf); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageWriteCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageWriteCacheTest.java index 6434b6f95e9..c38d16b877d 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageWriteCacheTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageWriteCacheTest.java @@ -34,6 +34,7 @@ import org.apache.bookkeeper.bookie.LedgerDirsManager; import org.apache.bookkeeper.bookie.TestBookieImpl; import org.apache.bookkeeper.bookie.storage.EntryLogger; +import org.apache.bookkeeper.conf.DbLedgerStorageConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; import org.apache.bookkeeper.meta.LedgerManager; @@ -111,8 +112,8 @@ public void setup() throws Exception { ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); conf.setGcWaitTime(gcWaitTime); conf.setLedgerStorageClass(MockedDbLedgerStorage.class.getName()); - conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 1); - conf.setProperty(DbLedgerStorage.MAX_THROTTLE_TIME_MILLIS, 1000); + conf.setProperty(DbLedgerStorageConfiguration.WRITE_CACHE_MAX_SIZE_MB, 1); + conf.setProperty(DbLedgerStorageConfiguration.MAX_THROTTLE_TIME_MILLIS, 1000); conf.setLedgerDirNames(new String[] { tmpDir.toString() }); Bookie bookie = new TestBookieImpl(conf); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java index 7617911e1d7..423649e9750 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java @@ -58,8 +58,8 @@ public static ServerConfiguration newServerConfiguration() { confReturn.setDiskUsageThreshold(0.999f); confReturn.setDiskUsageWarnThreshold(0.99f); confReturn.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap); - confReturn.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 4); - confReturn.setProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB, 4); + confReturn.setProperty(DbLedgerStorageConfiguration.WRITE_CACHE_MAX_SIZE_MB, 4); + confReturn.setProperty(DbLedgerStorageConfiguration.READ_AHEAD_CACHE_MAX_SIZE_MB, 4); /** * if testcase has zk error,just try 0 time for fast running */ From b0e2e7f67b3b7f6b0a10befe090f6910dcc24b85 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Fri, 29 Jul 2022 23:35:29 +0800 Subject: [PATCH 08/10] fix checkstyle --- .../ldb/SingleDirectoryDbLedgerStorage.java | 1 - .../conf/DbLedgerStorageConfiguration.java | 20 +++++++++++-------- .../bookkeeper/conf/ServerConfiguration.java | 18 ++++++++--------- .../org/apache/bookkeeper/server/Main.java | 2 -- .../bookkeeper/conf/TestBKConfiguration.java | 1 - 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 0445224ca5d..d8371dfd46e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -68,7 +68,6 @@ import org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorageDataFormats.LedgerData; import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorage.Batch; import org.apache.bookkeeper.common.util.Watcher; -import org.apache.bookkeeper.conf.DbLedgerStorageConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.proto.BookieProtocol; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java index 3c478db2831..9fbd83c1b1c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java @@ -20,14 +20,14 @@ */ package org.apache.bookkeeper.conf; +import java.util.concurrent.TimeUnit; + import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.NettyRuntime; import io.netty.util.internal.PlatformDependent; import org.apache.commons.lang3.StringUtils; -import java.util.concurrent.TimeUnit; - public class DbLedgerStorageConfiguration extends ServerConfiguration { private static final int MB = 1024 * 1024; @@ -50,24 +50,28 @@ public class DbLedgerStorageConfiguration extends ServerConfiguration { private static final String DIRECT_IO_ENTRYLOGGER = "dbStorage_directIOEntryLogger"; - private static final String DIRECT_IO_ENTRYLOGGER_TOTAL_WRITEBUFFER_SIZE_MB = "dbStorage_directIOEntryLoggerTotalWriteBufferSizeMb"; + private static final String DIRECT_IO_ENTRYLOGGER_TOTAL_WRITEBUFFER_SIZE_MB = + "dbStorage_directIOEntryLoggerTotalWriteBufferSizeMb"; private static final long DEFAULT_DIRECT_IO_TOTAL_WRITEBUFFER_SIZE_MB = (long) (0.125 * PlatformDependent.estimateMaxDirectMemory()) / MB; - private static final String DIRECT_IO_ENTRYLOGGER_TOTAL_READBUFFER_SIZE_MB = "dbStorage_directIOEntryLoggerTotalReadBufferSizeMb"; + private static final String DIRECT_IO_ENTRYLOGGER_TOTAL_READBUFFER_SIZE_MB = + "dbStorage_directIOEntryLoggerTotalReadBufferSizeMb"; private static final long DEFAULT_DIRECT_IO_TOTAL_READBUFFER_SIZE_MB = (long) (0.125 * PlatformDependent.estimateMaxDirectMemory()) / MB; - private static final String DIRECT_IO_ENTRYLOGGER_READBUFFER_SIZE_MB = "dbStorage_directIOEntryLoggerReadBufferSizeMb"; + private static final String DIRECT_IO_ENTRYLOGGER_READBUFFER_SIZE_MB = + "dbStorage_directIOEntryLoggerReadBufferSizeMb"; private static final long DEFAULT_DIRECT_IO_READBUFFER_SIZE_MB = 8; - private static final String DIRECT_IO_ENTRYLOGGER_MAX_FD_CACHE_TIME_SECONDS = "dbStorage_directIOEntryLoggerMaxFdCacheTimeSeconds"; + private static final String DIRECT_IO_ENTRYLOGGER_MAX_FD_CACHE_TIME_SECONDS = + "dbStorage_directIOEntryLoggerMaxFdCacheTimeSeconds"; private static final int DEFAULT_DIRECT_IO_MAX_FD_CACHE_TIME_SECONDS = 300; - + @VisibleForTesting public static final String MAX_THROTTLE_TIME_MILLIS = "dbStorage_maxThrottleTimeMs"; @@ -88,7 +92,7 @@ public int getReadAheadCacheBatchSize() { public boolean isDirectIOEntryLoggerEnabled() { return getBooleanVariableOrDefault(DIRECT_IO_ENTRYLOGGER, false); } - + public long getDirectIOEntryLoggerTotalWriteBufferSize() { return getLongVariableOrDefault(DIRECT_IO_ENTRYLOGGER_TOTAL_WRITEBUFFER_SIZE_MB, DEFAULT_DIRECT_IO_TOTAL_WRITEBUFFER_SIZE_MB) * MB; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index c56f5862739..3666fb0031d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -17,19 +17,12 @@ */ package org.apache.bookkeeper.conf; -import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; - import com.google.common.annotations.Beta; import com.google.common.base.Strings; import com.google.common.collect.Lists; - import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.NettyRuntime; import io.netty.util.internal.PlatformDependent; - -import java.io.File; -import java.net.URL; -import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.bookie.FileChannelProvider; import org.apache.bookkeeper.bookie.InterleavedLedgerStorage; import org.apache.bookkeeper.bookie.LedgerStorage; @@ -51,6 +44,12 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.lang3.StringUtils; +import java.io.File; +import java.net.URL; +import java.util.concurrent.TimeUnit; + +import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; + /** * Configuration manages server-side settings. */ @@ -337,7 +336,7 @@ public class ServerConfiguration extends AbstractConfiguration Date: Sat, 30 Jul 2022 00:48:51 +0800 Subject: [PATCH 09/10] fix checkstyle --- .../conf/DbLedgerStorageConfiguration.java | 5 ++-- .../bookkeeper/conf/ServerConfiguration.java | 30 +++++-------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java index 9fbd83c1b1c..0b52637501a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java @@ -20,12 +20,13 @@ */ package org.apache.bookkeeper.conf; -import java.util.concurrent.TimeUnit; - import com.google.common.annotations.VisibleForTesting; +// CHECKSTYLE.OFF: IllegalImport import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.NettyRuntime; import io.netty.util.internal.PlatformDependent; +// CHECKSTYLE.ON: IllegalImport +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; public class DbLedgerStorageConfiguration extends ServerConfiguration { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 3666fb0031d..393eb0b94c4 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -17,12 +17,15 @@ */ package org.apache.bookkeeper.conf; +import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; + import com.google.common.annotations.Beta; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import io.netty.buffer.PooledByteBufAllocator; -import io.netty.util.NettyRuntime; -import io.netty.util.internal.PlatformDependent; + +import java.io.File; +import java.net.URL; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.bookie.FileChannelProvider; import org.apache.bookkeeper.bookie.InterleavedLedgerStorage; import org.apache.bookkeeper.bookie.LedgerStorage; @@ -44,12 +47,6 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.lang3.StringUtils; -import java.io.File; -import java.net.URL; -import java.util.concurrent.TimeUnit; - -import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; - /** * Configuration manages server-side settings. */ @@ -4023,23 +4020,10 @@ public DbLedgerStorageConfiguration toDbLedgerStorageConfiguration() { return dbLedgerStorageConfiguration; } - /** - * The configured pooling concurrency for the allocator, if user config it, we should consider the unpooled - * direct memory which readCache and writeCache occupy when use DbLedgerStorage. - */ public int getAllocatorPoolingConcurrency() { String ledgerStorageClass = getLedgerStorageClass(); if (DbLedgerStorage.class.getName().equals(ledgerStorageClass)) { - DbLedgerStorageConfiguration dbLedgerStorageConfiguration = toDbLedgerStorageConfiguration(); - - long writeCacheSize = dbLedgerStorageConfiguration.getWriteCacheMaxSize(); - long readCacheSize = dbLedgerStorageConfiguration.getReadCacheMaxSize(); - long availableDirectMemory = PlatformDependent.maxDirectMemory() - writeCacheSize - readCacheSize; - - final int defaultChunkSize = - PooledByteBufAllocator.defaultPageSize() << PooledByteBufAllocator.defaultMaxOrder(); - int defaultMinNumArena = NettyRuntime.availableProcessors() * 2; - return Math.min(defaultMinNumArena, (int) (availableDirectMemory / defaultChunkSize / 2 / 3)); + return toDbLedgerStorageConfiguration().getAllocatorPoolingConcurrency(); } return super.getAllocatorPoolingConcurrency(); } From 5c33be7a56afcf9e543c8c5a2fd57e35259e443d Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sat, 30 Jul 2022 00:57:41 +0800 Subject: [PATCH 10/10] fix checkstyle --- .../apache/bookkeeper/conf/DbLedgerStorageConfiguration.java | 4 ++-- .../java/org/apache/bookkeeper/conf/ServerConfiguration.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java index 0b52637501a..d972b410120 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/DbLedgerStorageConfiguration.java @@ -20,14 +20,14 @@ */ package org.apache.bookkeeper.conf; -import com.google.common.annotations.VisibleForTesting; // CHECKSTYLE.OFF: IllegalImport +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.NettyRuntime; import io.netty.util.internal.PlatformDependent; -// CHECKSTYLE.ON: IllegalImport import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; +// CHECKSTYLE.ON: IllegalImport public class DbLedgerStorageConfiguration extends ServerConfiguration { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 393eb0b94c4..a9c6cc1466b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -22,7 +22,6 @@ import com.google.common.annotations.Beta; import com.google.common.base.Strings; import com.google.common.collect.Lists; - import java.io.File; import java.net.URL; import java.util.concurrent.TimeUnit;