From 4808e1026f4b90a0390b48dcaf82af52871f76f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E8=83=9C=E5=88=A9?= <48829688+shenshengli@users.noreply.github.com> Date: Mon, 28 Dec 2020 16:43:14 +0800 Subject: [PATCH 1/8] Create hbase-default.xml 'dfs.client.read.shortcircuit' should not be set in hbase-default.xml --- hbase-common/src/main/resources/hbase-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index def502a62cfc..3dd8a89eefd6 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -1463,7 +1463,7 @@ possible configurations would overwhelm and obscure the important. dfs.client.read.shortcircuit - false + If set to true, this configuration parameter enables short-circuit local reads. From d1443c0c690ecf7d379d4e423f2ec221a39bb430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E8=83=9C=E5=88=A9?= <48829688+shenshengli@users.noreply.github.com> Date: Wed, 6 Jan 2021 15:36:08 +0800 Subject: [PATCH 2/8] Create hdfs-site.xml For comparison. --- hbase-common/src/test/resources/hdfs-site.xml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 hbase-common/src/test/resources/hdfs-site.xml diff --git a/hbase-common/src/test/resources/hdfs-site.xml b/hbase-common/src/test/resources/hdfs-site.xml new file mode 100644 index 000000000000..b11d587a237a --- /dev/null +++ b/hbase-common/src/test/resources/hdfs-site.xml @@ -0,0 +1,32 @@ + + + + + + + dfs.client.read.shortcircuit + true + + If set to true, this configuration parameter enables short-circuit local + reads. + + + From a2f47199dfa200287751d12db63f86cde8922ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E8=83=9C=E5=88=A9?= <48829688+shenshengli@users.noreply.github.com> Date: Wed, 6 Jan 2021 15:39:50 +0800 Subject: [PATCH 3/8] Create hdfs-default.xml For comparison. --- .../src/test/resources/hdfs-default.xml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 hbase-common/src/test/resources/hdfs-default.xml diff --git a/hbase-common/src/test/resources/hdfs-default.xml b/hbase-common/src/test/resources/hdfs-default.xml new file mode 100644 index 000000000000..11da91b44e59 --- /dev/null +++ b/hbase-common/src/test/resources/hdfs-default.xml @@ -0,0 +1,32 @@ + + + + + + + dfs.client.read.shortcircuit + false + + If set to true, this configuration parameter enables short-circuit local + reads. + + + From e81dba9b45d7d40f658d0d6ce065b3db390b953e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E8=83=9C=E5=88=A9?= <48829688+shenshengli@users.noreply.github.com> Date: Wed, 6 Jan 2021 15:52:18 +0800 Subject: [PATCH 4/8] Create TestHBaseConfiguration.java Adding Unit Tests.Make this configuration empty in hbase-default.xml , get values in hdfs-default.xml or hdfs-site.xml --- .../apache/hadoop/hbase/TestHBaseConfiguration.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestHBaseConfiguration.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestHBaseConfiguration.java index 6a0b4283ed03..96213e08ce78 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestHBaseConfiguration.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestHBaseConfiguration.java @@ -115,6 +115,19 @@ public void testSecurityConfCaseInsensitive() { conf.set("hbase.security.authentication", "KERBeros"); Assert.assertTrue(User.isHBaseSecurityEnabled(conf)); } + + @Test + public void testGetConfigOfShortcircuitRead() throws Exception { + Configuration conf = HBaseConfiguration.create(); + Configuration.addDefaultResource("hdfs-default.xml"); + assertEquals("hdfs-default.xml", + conf.getPropertySources("dfs.client.read.shortcircuit")[0]); + assertEquals("false", conf.get("dfs.client.read.shortcircuit")); + Configuration.addDefaultResource("hdfs-site.xml"); + assertEquals("hdfs-site.xml", + conf.getPropertySources("dfs.client.read.shortcircuit")[0]); + assertEquals("true", conf.get("dfs.client.read.shortcircuit")); + } private static class ReflectiveCredentialProviderClient { public static final String HADOOP_CRED_PROVIDER_FACTORY_CLASS_NAME = From 33a6f26a8fd2aba3b33a06260b67e34cafbb2ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E8=83=9C=E5=88=A9?= <48829688+shenshengli@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:26:47 +0800 Subject: [PATCH 5/8] Create hbase-default.xml Set the parameter "dfs.domain.socket.path" to null as well --- hbase-common/src/main/resources/hbase-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 3dd8a89eefd6..f133ed6c95e5 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -1471,7 +1471,7 @@ possible configurations would overwhelm and obscure the important. dfs.domain.socket.path - none + This is a path to a UNIX domain socket that will be used for communication between the DataNode and local HDFS clients, if From 7b5cb53acd3f235b14f7e46a6ae8926185d1b477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E8=83=9C=E5=88=A9?= <48829688+shenshengli@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:34:07 +0800 Subject: [PATCH 6/8] Create hdfs-default.xml --- hbase-common/src/test/resources/hdfs-default.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hbase-common/src/test/resources/hdfs-default.xml b/hbase-common/src/test/resources/hdfs-default.xml index 11da91b44e59..fdb3c36edc87 100644 --- a/hbase-common/src/test/resources/hdfs-default.xml +++ b/hbase-common/src/test/resources/hdfs-default.xml @@ -29,4 +29,14 @@ reads. + + dfs.domain.socket.path + + + Optional. This is a path to a UNIX domain socket that will be used for + communication between the DataNode and local HDFS clients. + If the string "_PORT" is present in this path, it will be replaced by the + TCP port of the DataNode. + + From 6552c65cab71b29c680a2ffaf3b5988b7b4f4de6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E8=83=9C=E5=88=A9?= <48829688+shenshengli@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:38:44 +0800 Subject: [PATCH 7/8] Create hdfs-scr-enabled.xml --- .../resources/{hdfs-site.xml => hdfs-scr-enabled.xml} | 10 ++++++++++ 1 file changed, 10 insertions(+) rename hbase-common/src/test/resources/{hdfs-site.xml => hdfs-scr-enabled.xml} (73%) diff --git a/hbase-common/src/test/resources/hdfs-site.xml b/hbase-common/src/test/resources/hdfs-scr-enabled.xml similarity index 73% rename from hbase-common/src/test/resources/hdfs-site.xml rename to hbase-common/src/test/resources/hdfs-scr-enabled.xml index b11d587a237a..8594494782c5 100644 --- a/hbase-common/src/test/resources/hdfs-site.xml +++ b/hbase-common/src/test/resources/hdfs-scr-enabled.xml @@ -29,4 +29,14 @@ reads. + + dfs.domain.socket.path + /var/lib/hadoop-hdfs/dn_socket + + Optional. This is a path to a UNIX domain socket that will be used for + communication between the DataNode and local HDFS clients. + If the string "_PORT" is present in this path, it will be replaced by the + TCP port of the DataNode. + + From 8d19f3d7d32c5d7d3321c46191655fa47244dbe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E8=83=9C=E5=88=A9?= <48829688+shenshengli@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:46:49 +0800 Subject: [PATCH 8/8] Create TestHBaseConfiguration.java --- .../org/apache/hadoop/hbase/TestHBaseConfiguration.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestHBaseConfiguration.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestHBaseConfiguration.java index 96213e08ce78..1144f1daf351 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestHBaseConfiguration.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestHBaseConfiguration.java @@ -123,10 +123,14 @@ public void testGetConfigOfShortcircuitRead() throws Exception { assertEquals("hdfs-default.xml", conf.getPropertySources("dfs.client.read.shortcircuit")[0]); assertEquals("false", conf.get("dfs.client.read.shortcircuit")); - Configuration.addDefaultResource("hdfs-site.xml"); - assertEquals("hdfs-site.xml", + assertNull(conf.get("dfs.domain.socket.path")); + Configuration.addDefaultResource("hdfs-scr-enabled.xml"); + assertEquals("hdfs-scr-enabled.xml", conf.getPropertySources("dfs.client.read.shortcircuit")[0]); + assertEquals("hdfs-scr-enabled.xml", + conf.getPropertySources("dfs.domain.socket.path")[0]); assertEquals("true", conf.get("dfs.client.read.shortcircuit")); + assertEquals("/var/lib/hadoop-hdfs/dn_socket", conf.get("dfs.domain.socket.path")); } private static class ReflectiveCredentialProviderClient {