+ * Over time, as a BucketSizeInfo gets more allocations, it will claim more buckets from the total + * pool of completelyFreeBuckets. As blocks are freed from a BucketSizeInfo, those buckets may be + * returned to the completelyFreeBuckets pool. + *
+ * The IndexStatistics help visualize how these buckets are currently distributed, through counts + * of items, bytes, and fullBuckets. Additionally, mismatches between block sizes and bucket sizes + * can manifest in inefficient cache usage. These typically manifest in three ways: + *
+ * 1. Allocation failures, because block size is larger than max bucket size. These show up in
+ * logs and can be alleviated by adding larger bucket sizes if appropriate.
+ * 2. Memory fragmentation, because blocks are typically smaller than the bucket size. See
+ * {@link #fragmentationBytes()} for details.
+ * 3. Memory waste, because a bucket's itemSize is not a perfect divisor of bucketCapacity. see
+ * {@link #wastedBytes()} for details.
+ */
static class IndexStatistics {
- private long freeCount, usedCount, itemSize, totalCount;
+ private long freeCount, usedCount, itemSize, totalCount, wastedBytes, fragmentationBytes;
+ private int fullBuckets, completelyFreeBuckets;
+ /**
+ * How many more items can be allocated from the currently claimed blocks of this bucket size
+ */
public long freeCount() {
return freeCount;
}
+ /**
+ * How many items are currently taking up space in this bucket size's buckets
+ */
public long usedCount() {
return usedCount;
}
+ /**
+ * Combined {@link #freeCount()} + {@link #usedCount()}
+ */
public long totalCount() {
return totalCount;
}
+ /**
+ * How many more bytes can be allocated from the currently claimed blocks of this bucket size
+ */
public long freeBytes() {
return freeCount * itemSize;
}
+ /**
+ * How many bytes are currently taking up space in this bucket size's buckets Note: If your
+ * items are less than the bucket size of this bucket, the actual used bytes by items will be
+ * lower than this value. But since a bucket size can only allocate items of a single size, this
+ * value is the true number of used bytes. The difference will be counted in
+ * {@link #fragmentationBytes()}.
+ */
public long usedBytes() {
return usedCount * itemSize;
}
+ /**
+ * Combined {@link #totalCount()} * {@link #itemSize()}
+ */
public long totalBytes() {
return totalCount * itemSize;
}
+ /**
+ * This bucket size can only allocate items of this size, even if the requested allocation size
+ * is smaller. The rest goes towards {@link #fragmentationBytes()}.
+ */
public long itemSize() {
return itemSize;
}
- public IndexStatistics(long free, long used, long itemSize) {
- setTo(free, used, itemSize);
+ /**
+ * How many buckets have been completely filled by blocks for this bucket size. These buckets
+ * can't accept any more blocks unless some existing are freed.
+ */
+ public int fullBuckets() {
+ return fullBuckets;
+ }
+
+ /**
+ * How many buckets are currently claimed by this bucket size but as yet totally unused. These
+ * buckets are available for reallocation to other bucket sizes if those fill up.
+ */
+ public int completelyFreeBuckets() {
+ return completelyFreeBuckets;
+ }
+
+ /**
+ * If {@link #bucketCapacity} is not perfectly divisible by this {@link #itemSize()}, the
+ * remainder will be unusable by in buckets of this size. A high value here may be optimized by
+ * trying to choose bucket sizes which can better divide {@link #bucketCapacity}.
+ */
+ public long wastedBytes() {
+ return wastedBytes;
+ }
+
+ /**
+ * Every time you allocate blocks in these buckets where the block size is less than the bucket
+ * size, fragmentation increases by that difference. You can reduce fragmentation by lowering
+ * the bucket size so that it is closer to the typical block size. This may have the consequence
+ * of bumping some blocks to the next larger bucket size, so experimentation may be needed.
+ */
+ public long fragmentationBytes() {
+ return fragmentationBytes;
+ }
+
+ public IndexStatistics(long free, long used, long itemSize, int fullBuckets,
+ int completelyFreeBuckets, long wastedBytes, long fragmentationBytes) {
+ setTo(free, used, itemSize, fullBuckets, completelyFreeBuckets, wastedBytes,
+ fragmentationBytes);
}
public IndexStatistics() {
- setTo(-1, -1, 0);
+ setTo(-1, -1, 0, 0, 0, 0, 0);
}
- public void setTo(long free, long used, long itemSize) {
+ public void setTo(long free, long used, long itemSize, int fullBuckets,
+ int completelyFreeBuckets, long wastedBytes, long fragmentationBytes) {
this.itemSize = itemSize;
this.freeCount = free;
this.usedCount = used;
this.totalCount = free + used;
+ this.fullBuckets = fullBuckets;
+ this.completelyFreeBuckets = completelyFreeBuckets;
+ this.wastedBytes = wastedBytes;
+ this.fragmentationBytes = fragmentationBytes;
}
}
@@ -529,26 +641,43 @@ public Bucket[] getBuckets() {
return this.buckets;
}
- void logStatistics() {
+ void logDebugStatistics() {
+ if (!LOG.isDebugEnabled()) {
+ return;
+ }
+
IndexStatistics total = new IndexStatistics();
IndexStatistics[] stats = getIndexStatistics(total);
- LOG.info("Bucket allocator statistics follow:\n");
- LOG.info(" Free bytes=" + total.freeBytes() + "+; used bytes=" + total.usedBytes()
- + "; total bytes=" + total.totalBytes());
+ LOG.debug("Bucket allocator statistics follow:");
+ LOG.debug(
+ " Free bytes={}; used bytes={}; total bytes={}; wasted bytes={}; fragmentation bytes={}; "
+ + "completelyFreeBuckets={}",
+ total.freeBytes(), total.usedBytes(), total.totalBytes(), total.wastedBytes(),
+ total.fragmentationBytes(), total.completelyFreeBuckets());
for (IndexStatistics s : stats) {
- LOG.info(" Object size " + s.itemSize() + " used=" + s.usedCount() + "; free="
- + s.freeCount() + "; total=" + s.totalCount());
+ LOG.debug(
+ " Object size {}; used={}; free={}; total={}; wasted bytes={}; fragmentation bytes={}, "
+ + "full buckets={}",
+ s.itemSize(), s.usedCount(), s.freeCount(), s.totalCount(), s.wastedBytes(),
+ s.fragmentationBytes(), s.fullBuckets());
}
}
IndexStatistics[] getIndexStatistics(IndexStatistics grandTotal) {
IndexStatistics[] stats = getIndexStatistics();
- long totalfree = 0, totalused = 0;
+ long totalfree = 0, totalused = 0, totalWasted = 0, totalFragmented = 0;
+ int fullBuckets = 0, completelyFreeBuckets = 0;
+
for (IndexStatistics stat : stats) {
totalfree += stat.freeBytes();
totalused += stat.usedBytes();
+ totalWasted += stat.wastedBytes();
+ totalFragmented += stat.fragmentationBytes();
+ fullBuckets += stat.fullBuckets();
+ completelyFreeBuckets += stat.completelyFreeBuckets();
}
- grandTotal.setTo(totalfree, totalused, 1);
+ grandTotal.setTo(totalfree, totalused, 1, fullBuckets, completelyFreeBuckets, totalWasted,
+ totalFragmented);
return stats;
}
@@ -559,13 +688,6 @@ IndexStatistics[] getIndexStatistics() {
return stats;
}
- public long freeBlock(long freeList[]) {
- long sz = 0;
- for (int i = 0; i < freeList.length; ++i)
- sz += freeBlock(freeList[i]);
- return sz;
- }
-
public int getBucketIndex(long offset) {
return (int) (offset / bucketCapacity);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index c9a940768ab5..b30efd53fa79 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -580,7 +580,7 @@ void blockEvicted(BlockCacheKey cacheKey, BucketEntry bucketEntry, boolean decre
* {@link BucketEntry#refCnt} becoming 0.
*/
void freeBucketEntry(BucketEntry bucketEntry) {
- bucketAllocator.freeBlock(bucketEntry.offset());
+ bucketAllocator.freeBlock(bucketEntry.offset(), bucketEntry.getLength());
realCacheSize.add(-1 * bucketEntry.getLength());
}
@@ -738,6 +738,8 @@ public void logStats() {
+ cacheStats.getEvictedCount() + ", " + "evictedPerRun=" + cacheStats.evictedPerEviction()
+ ", " + "allocationFailCount=" + cacheStats.getAllocationFailCount());
cacheStats.reset();
+
+ bucketAllocator.logDebugStatistics();
}
public long getRealCacheSize() {
@@ -1119,8 +1121,9 @@ void doDrain(final List