From b09062fd4396355358c4c307425d32e4e126656c Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 18 May 2023 16:26:36 -0700 Subject: [PATCH 1/3] HBASE-27872 xerial's snappy-java requires GLIBC >= 2.32 We need to add a native library load check with a helpful error message if xerial snappy fails to initialize due to a too old glibc or similar reason, and disable the unit test if the native library fails to load. --- .../hbase/io/compress/xerial/SnappyCodec.java | 18 +++++++++++++++ .../xerial/TestHFileCompressionSnappy.java | 10 +++++++++ .../io/compress/xerial/TestSnappyCodec.java | 22 +++++++++++++++++++ .../xerial/TestWALCompressionSnappy.java | 14 ++++++++++++ 4 files changed, 64 insertions(+) diff --git a/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java b/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java index b6806dcbeefe..d151a8a9afd0 100644 --- a/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java +++ b/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java @@ -31,6 +31,8 @@ import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xerial.snappy.Snappy; /** @@ -43,8 +45,24 @@ public class SnappyCodec implements Configurable, CompressionCodec { public static final String SNAPPY_BUFFER_SIZE_KEY = "hbase.io.compress.snappy.buffersize"; + private static final Logger LOG = LoggerFactory.getLogger(SnappyCodec.class); private Configuration conf; private int bufferSize; + private static boolean loaded = false; + + static { + try { + Snappy.getNativeLibraryVersion(); + loaded = true; + } catch (Throwable t) { + LOG.error("The Snappy native libraries could not be loaded", t); + } + } + + /** Return true if the native shared libraries were loaded; false otherwise. */ + public static boolean isLoaded() { + return loaded; + } public SnappyCodec() { conf = new Configuration(); diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java index 42b0735dd26b..52bfe9384519 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java @@ -29,6 +29,8 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category({ IOTests.class, SmallTests.class }) public class TestHFileCompressionSnappy extends HFileTestBase { @@ -37,10 +39,15 @@ public class TestHFileCompressionSnappy extends HFileTestBase { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestHFileCompressionSnappy.class); + private static final Logger LOG = LoggerFactory.getLogger(TestHFileCompressionSnappy.class); private static Configuration conf; @BeforeClass public static void setUpBeforeClass() throws Exception { + if (!SnappyCodec.isLoaded()) { + LOG.warn("Snappy codec cannot be loaded. Test will not execute."); + return; + } conf = TEST_UTIL.getConfiguration(); conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName()); Compression.Algorithm.SNAPPY.reload(conf); @@ -49,6 +56,9 @@ public static void setUpBeforeClass() throws Exception { @Test public void test() throws Exception { + if (!SnappyCodec.isLoaded()) { + return; + } Path path = new Path(TEST_UTIL.getDataTestDir(), HBaseTestingUtil.getRandomUUID().toString() + ".hfile"); doTest(conf, path, Compression.Algorithm.SNAPPY); diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java index f62c35956736..143bcf3b5eb1 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java @@ -20,9 +20,12 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.io.compress.CompressionTestBase; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category(SmallTests.class) public class TestSnappyCodec extends CompressionTestBase { @@ -31,13 +34,29 @@ public class TestSnappyCodec extends CompressionTestBase { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestSnappyCodec.class); + private static final Logger LOG = LoggerFactory.getLogger(TestSnappyCodec.class); + + @BeforeClass + public static void setupClass() throws Exception { + if (!SnappyCodec.isLoaded()) { + LOG.warn("Snappy codec cannot be loaded. Test will not execute."); + return; + } + } + @Test public void testSnappyCodecSmall() throws Exception { + if (!SnappyCodec.isLoaded()) { + return; + } codecSmallTest(new SnappyCodec()); } @Test public void testSnappyCodecLarge() throws Exception { + if (!SnappyCodec.isLoaded()) { + return; + } codecLargeTest(new SnappyCodec(), 1.1); // poor compressability codecLargeTest(new SnappyCodec(), 2); codecLargeTest(new SnappyCodec(), 10); // very high compressability @@ -45,6 +64,9 @@ public void testSnappyCodecLarge() throws Exception { @Test public void testSnappyCodecVeryLarge() throws Exception { + if (!SnappyCodec.isLoaded()) { + return; + } codecVeryLargeTest(new SnappyCodec(), 3); // like text } diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java index dfbb63d0f6cc..ad6785e203c6 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java @@ -33,10 +33,14 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category({ RegionServerTests.class, MediumTests.class }) public class TestWALCompressionSnappy extends CompressedWALTestBase { + private static final Logger LOG = LoggerFactory.getLogger(TestWALCompressionSnappy.class); + @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestWALCompressionSnappy.class); @@ -46,6 +50,10 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase { @BeforeClass public static void setUpBeforeClass() throws Exception { + if (!SnappyCodec.isLoaded()) { + LOG.warn("Snappy codec cannot be loaded. Test will not execute."); + return; + } Configuration conf = TEST_UTIL.getConfiguration(); conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName()); Compression.Algorithm.SNAPPY.reload(conf); @@ -57,11 +65,17 @@ public static void setUpBeforeClass() throws Exception { @AfterClass public static void tearDown() throws Exception { + if (!SnappyCodec.isLoaded()) { + return; + } TEST_UTIL.shutdownMiniCluster(); } @Test public void test() throws Exception { + if (!SnappyCodec.isLoaded()) { + return; + } TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_")); doTest(tableName); } From de5c6ae677fbefead12fd8e7442c0085ca307f3c Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 18 May 2023 17:01:02 -0700 Subject: [PATCH 2/3] Also prevent the instantiation of SnappyCodec if the native libraries failed to load --- .../apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java b/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java index d151a8a9afd0..917676be2fe1 100644 --- a/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java +++ b/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java @@ -49,12 +49,14 @@ public class SnappyCodec implements Configurable, CompressionCodec { private Configuration conf; private int bufferSize; private static boolean loaded = false; + private static Throwable loadError; static { try { Snappy.getNativeLibraryVersion(); loaded = true; } catch (Throwable t) { + loadError = t; LOG.error("The Snappy native libraries could not be loaded", t); } } @@ -65,6 +67,9 @@ public static boolean isLoaded() { } public SnappyCodec() { + if (!isLoaded()) { + throw new RuntimeException("Snappy codec could not be loaded", loadError); + } conf = new Configuration(); bufferSize = getBufferSize(conf); } From c9e05c7d096b56d47514103b3351d2ba28e4e3a6 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 19 May 2023 14:39:17 -0700 Subject: [PATCH 3/3] Address review feedback --- .../xerial/TestHFileCompressionSnappy.java | 13 +++--------- .../io/compress/xerial/TestSnappyCodec.java | 20 +++---------------- .../xerial/TestWALCompressionSnappy.java | 17 +++------------- 3 files changed, 9 insertions(+), 41 deletions(-) diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java index 52bfe9384519..53c7c819a281 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.io.compress.xerial; +import static org.junit.Assume.assumeTrue; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -29,8 +31,6 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Category({ IOTests.class, SmallTests.class }) public class TestHFileCompressionSnappy extends HFileTestBase { @@ -39,15 +39,11 @@ public class TestHFileCompressionSnappy extends HFileTestBase { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestHFileCompressionSnappy.class); - private static final Logger LOG = LoggerFactory.getLogger(TestHFileCompressionSnappy.class); private static Configuration conf; @BeforeClass public static void setUpBeforeClass() throws Exception { - if (!SnappyCodec.isLoaded()) { - LOG.warn("Snappy codec cannot be loaded. Test will not execute."); - return; - } + assumeTrue(SnappyCodec.isLoaded()); conf = TEST_UTIL.getConfiguration(); conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName()); Compression.Algorithm.SNAPPY.reload(conf); @@ -56,9 +52,6 @@ public static void setUpBeforeClass() throws Exception { @Test public void test() throws Exception { - if (!SnappyCodec.isLoaded()) { - return; - } Path path = new Path(TEST_UTIL.getDataTestDir(), HBaseTestingUtil.getRandomUUID().toString() + ".hfile"); doTest(conf, path, Compression.Algorithm.SNAPPY); diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java index 143bcf3b5eb1..8c011f9f6892 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.io.compress.xerial; +import static org.junit.Assume.assumeTrue; + import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.io.compress.CompressionTestBase; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -24,8 +26,6 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Category(SmallTests.class) public class TestSnappyCodec extends CompressionTestBase { @@ -34,29 +34,18 @@ public class TestSnappyCodec extends CompressionTestBase { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestSnappyCodec.class); - private static final Logger LOG = LoggerFactory.getLogger(TestSnappyCodec.class); - @BeforeClass public static void setupClass() throws Exception { - if (!SnappyCodec.isLoaded()) { - LOG.warn("Snappy codec cannot be loaded. Test will not execute."); - return; - } + assumeTrue(SnappyCodec.isLoaded()); } @Test public void testSnappyCodecSmall() throws Exception { - if (!SnappyCodec.isLoaded()) { - return; - } codecSmallTest(new SnappyCodec()); } @Test public void testSnappyCodecLarge() throws Exception { - if (!SnappyCodec.isLoaded()) { - return; - } codecLargeTest(new SnappyCodec(), 1.1); // poor compressability codecLargeTest(new SnappyCodec(), 2); codecLargeTest(new SnappyCodec(), 10); // very high compressability @@ -64,9 +53,6 @@ public void testSnappyCodecLarge() throws Exception { @Test public void testSnappyCodecVeryLarge() throws Exception { - if (!SnappyCodec.isLoaded()) { - return; - } codecVeryLargeTest(new SnappyCodec(), 3); // like text } diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java index ad6785e203c6..7a2bbfe7b872 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.io.compress.xerial; +import static org.junit.Assume.assumeTrue; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; @@ -33,14 +35,10 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Category({ RegionServerTests.class, MediumTests.class }) public class TestWALCompressionSnappy extends CompressedWALTestBase { - private static final Logger LOG = LoggerFactory.getLogger(TestWALCompressionSnappy.class); - @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestWALCompressionSnappy.class); @@ -50,10 +48,7 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase { @BeforeClass public static void setUpBeforeClass() throws Exception { - if (!SnappyCodec.isLoaded()) { - LOG.warn("Snappy codec cannot be loaded. Test will not execute."); - return; - } + assumeTrue(SnappyCodec.isLoaded()); Configuration conf = TEST_UTIL.getConfiguration(); conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName()); Compression.Algorithm.SNAPPY.reload(conf); @@ -65,17 +60,11 @@ public static void setUpBeforeClass() throws Exception { @AfterClass public static void tearDown() throws Exception { - if (!SnappyCodec.isLoaded()) { - return; - } TEST_UTIL.shutdownMiniCluster(); } @Test public void test() throws Exception { - if (!SnappyCodec.isLoaded()) { - return; - } TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_")); doTest(tableName); }