From 39e5799fa366cac0c05f668d28e8ca127ff44296 Mon Sep 17 00:00:00 2001 From: "Jain, Nihal" Date: Wed, 14 Feb 2024 18:22:45 +0530 Subject: [PATCH 1/2] HBASE-28368 Backport "HBASE-27693 Support for Hadoop's LDAP Authentication mechanism (Web UI only)" to branch-2 Co-authored-by: Yash Dodeja --- hbase-http/pom.xml | 60 ++++++++ .../lib/AuthenticationFilterInitializer.java | 90 +++++++++++ .../hadoop/hbase/http/LdapConstants.java | 35 +++++ .../hadoop/hbase/http/TestLdapHttpServer.java | 140 ++++++++++++++++++ .../TestAuthenticationFilterInitializer.java | 80 ++++++++++ pom.xml | 3 + 6 files changed, 408 insertions(+) create mode 100644 hbase-http/src/main/java/org/apache/hadoop/hbase/http/lib/AuthenticationFilterInitializer.java create mode 100644 hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapConstants.java create mode 100644 hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java create mode 100644 hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestAuthenticationFilterInitializer.java diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 497fdf3b3719..7a8f37521fb1 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -147,6 +147,60 @@ log4j-1.2-api test + + org.apache.directory.server + apacheds-core + ${apacheds.version} + test + + + org.bouncycastle + bcprov-jdk15on + + + + + org.apache.directory.server + apacheds-protocol-ldap + ${apacheds.version} + test + + + org.bouncycastle + bcprov-jdk15on + + + + + org.apache.directory.server + apacheds-ldif-partition + ${apacheds.version} + test + + + org.apache.directory.api + api-ldap-codec-core + ${ldap-api.version} + test + + + org.apache.directory.api + api-ldap-model + ${ldap-api.version} + test + + + org.apache.directory.server + apacheds-server-integ + ${apacheds.version} + test + + + log4j + log4j + + + @@ -357,6 +411,12 @@ org.apache.hadoop hadoop-minikdc test + + + org.apache.directory.api + api-all + + org.apache.kerby diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/lib/AuthenticationFilterInitializer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/lib/AuthenticationFilterInitializer.java new file mode 100644 index 000000000000..f3a10d6a54e3 --- /dev/null +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/lib/AuthenticationFilterInitializer.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.http.lib; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.http.FilterContainer; +import org.apache.hadoop.hbase.http.FilterInitializer; +import org.apache.hadoop.hbase.http.HttpServer; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * This class is copied from Hadoop. Initializes hadoop-auth AuthenticationFilter which provides + * support for Kerberos HTTP SPNEGO authentication. + *

+ * It enables anonymous access, simple/pseudo and Kerberos HTTP SPNEGO authentication for HBase web + * UI endpoints. + *

+ * Refer to the core-default.xml file, after the comment 'HTTP Authentication' for + * details on the configuration options. All related configuration properties have + * 'hadoop.http.authentication.' as prefix. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) +public class AuthenticationFilterInitializer extends FilterInitializer { + + static final String PREFIX = "hadoop.http.authentication."; + + /** + * Initializes hadoop-auth AuthenticationFilter. + *

+ * Propagates to hadoop-auth AuthenticationFilter configuration all Hadoop configuration + * properties prefixed with "hadoop.http.authentication." + * @param container The filter container + * @param conf Configuration for run-time parameters + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + Map filterConfig = getFilterConfigMap(conf, PREFIX); + + container.addFilter("authentication", AuthenticationFilter.class.getName(), filterConfig); + } + + public static Map getFilterConfigMap(Configuration conf, String prefix) { + Map filterConfig = new HashMap(); + + // setting the cookie path to root '/' so it is used for all resources. + filterConfig.put(AuthenticationFilter.COOKIE_PATH, "/"); + Map propsWithPrefix = conf.getPropsWithPrefix(prefix); + + for (Map.Entry entry : propsWithPrefix.entrySet()) { + filterConfig.put(entry.getKey(), entry.getValue()); + } + + // Resolve _HOST into bind address + String bindAddress = conf.get(HttpServer.BIND_ADDRESS); + String principal = filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL); + if (principal != null) { + try { + principal = SecurityUtil.getServerPrincipal(principal, bindAddress); + } catch (IOException ex) { + throw new RuntimeException("Could not resolve Kerberos principal name: " + ex.toString(), + ex); + } + filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, principal); + } + return filterConfig; + } + +} diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapConstants.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapConstants.java new file mode 100644 index 000000000000..a73aaa810c3d --- /dev/null +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapConstants.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.http; + +/** + * This class defines the constants used by the LDAP integration tests. + */ +public final class LdapConstants { + + /** + * This class defines constants to be used for LDAP integration testing. Hence this class is not + * expected to be instantiated. + */ + private LdapConstants() { + } + + public static final String LDAP_BASE_DN = "dc=example,dc=com"; + public static final String LDAP_SERVER_ADDR = "localhost"; + +} diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java new file mode 100644 index 000000000000..8bb48d50753e --- /dev/null +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.http; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import org.apache.commons.codec.binary.Base64; +import org.apache.directory.server.annotations.CreateLdapServer; +import org.apache.directory.server.annotations.CreateTransport; +import org.apache.directory.server.core.annotations.ApplyLdifs; +import org.apache.directory.server.core.annotations.ContextEntry; +import org.apache.directory.server.core.annotations.CreateDS; +import org.apache.directory.server.core.annotations.CreatePartition; +import org.apache.directory.server.core.integ.CreateLdapServerRule; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.http.resource.JerseyResource; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test class for LDAP authentication on the HttpServer. + */ +@Category({ MiscTests.class, SmallTests.class }) +@CreateLdapServer( + transports = { @CreateTransport(protocol = "LDAP", address = LdapConstants.LDAP_SERVER_ADDR), }) +@CreateDS(allowAnonAccess = true, + partitions = { @CreatePartition(name = "Test_Partition", suffix = LdapConstants.LDAP_BASE_DN, + contextEntry = @ContextEntry(entryLdif = "dn: " + LdapConstants.LDAP_BASE_DN + " \n" + + "dc: example\n" + "objectClass: top\n" + "objectClass: domain\n\n")) }) +@ApplyLdifs({ "dn: uid=bjones," + LdapConstants.LDAP_BASE_DN, "cn: Bob Jones", "sn: Jones", + "objectClass: inetOrgPerson", "uid: bjones", "userPassword: p@ssw0rd" }) +public class TestLdapHttpServer extends HttpServerFunctionalTest { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestLdapHttpServer.class); + @ClassRule + public static CreateLdapServerRule serverRule = new CreateLdapServerRule(); + + private static final Logger LOG = LoggerFactory.getLogger(TestLdapHttpServer.class); + + private static HttpServer server; + private static URL baseUrl; + + @BeforeClass + public static void setupServer() throws Exception { + Configuration conf = new Configuration(); + buildLdapConfiguration(conf); + server = createTestServer(conf); + server.addUnprivilegedServlet("echo", "/echo", TestHttpServer.EchoServlet.class); + server.addJerseyResourcePackage(JerseyResource.class.getPackage().getName(), "/jersey/*"); + server.start(); + baseUrl = getServerURL(server); + + LOG.info("HTTP server started: " + baseUrl); + } + + @AfterClass + public static void stopServer() throws Exception { + try { + if (null != server) { + server.stop(); + } + } catch (Exception e) { + LOG.info("Failed to stop info server", e); + } + } + + private static Configuration buildLdapConfiguration(Configuration conf) { + + conf.setInt(HttpServer.HTTP_MAX_THREADS, TestHttpServer.MAX_THREADS); + + // Enable LDAP (pre-req) + conf.set(HttpServer.HTTP_UI_AUTHENTICATION, "ldap"); + conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY, + "org.apache.hadoop.hbase.http.lib.AuthenticationFilterInitializer"); + conf.set("hadoop.http.authentication.type", "ldap"); + conf.set("hadoop.http.authentication.ldap.providerurl", String.format("ldap://%s:%s", + LdapConstants.LDAP_SERVER_ADDR, serverRule.getLdapServer().getPort())); + conf.set("hadoop.http.authentication.ldap.enablestarttls", "false"); + conf.set("hadoop.http.authentication.ldap.basedn", LdapConstants.LDAP_BASE_DN); + return conf; + } + + @Test + public void testUnauthorizedClientsDisallowed() throws IOException { + URL url = new URL(getServerURL(server), "/echo?a=b"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, conn.getResponseCode()); + } + + @Test + public void testAllowedClient() throws IOException { + URL url = new URL(getServerURL(server), "/echo?a=b"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + final Base64 base64 = new Base64(0); + String userCredentials = "bjones:p@ssw0rd"; + String basicAuth = "Basic " + base64.encodeToString(userCredentials.getBytes()); + conn.setRequestProperty("Authorization", basicAuth); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + } + + @Test + public void testWrongAuthClientsDisallowed() throws IOException { + URL url = new URL(getServerURL(server), "/echo?a=b"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + final Base64 base64 = new Base64(0); + String userCredentials = "bjones:password"; + String basicAuth = "Basic " + base64.encodeToString(userCredentials.getBytes()); + conn.setRequestProperty("Authorization", basicAuth); + assertEquals(HttpURLConnection.HTTP_FORBIDDEN, conn.getResponseCode()); + } + +} diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestAuthenticationFilterInitializer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestAuthenticationFilterInitializer.java new file mode 100644 index 000000000000..68c48f282737 --- /dev/null +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestAuthenticationFilterInitializer.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.http.lib; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; + +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.http.FilterContainer; +import org.apache.hadoop.hbase.http.HttpServer; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +@Category({ MiscTests.class, SmallTests.class }) +public class TestAuthenticationFilterInitializer { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestAuthenticationFilterInitializer.class); + + @Test + public void testConfiguration() throws Exception { + Configuration conf = new Configuration(); + conf.set("hadoop.http.authentication.foo", "bar"); + + conf.set(HttpServer.BIND_ADDRESS, "barhost"); + + FilterContainer container = Mockito.mock(FilterContainer.class); + Mockito.doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + Object[] args = invocationOnMock.getArguments(); + + assertEquals("authentication", args[0]); + + assertEquals(AuthenticationFilter.class.getName(), args[1]); + + Map conf = (Map) args[2]; + assertEquals("/", conf.get("cookie.path")); + + assertEquals("simple", conf.get("type")); + assertEquals("36000", conf.get("token.validity")); + assertNull(conf.get("cookie.domain")); + assertEquals("true", conf.get("simple.anonymous.allowed")); + assertEquals("HTTP/barhost@LOCALHOST", conf.get("kerberos.principal")); + assertEquals(System.getProperty("user.home") + "/hadoop.keytab", + conf.get("kerberos.keytab")); + assertEquals("bar", conf.get("foo")); + + return null; + } + }).when(container).addFilter(any(), any(), any()); + + new AuthenticationFilterInitializer().initFilter(container, conf); + } + +} diff --git a/pom.xml b/pom.xml index da922ff5f2c3..a666a71e83b3 100644 --- a/pom.xml +++ b/pom.xml @@ -786,6 +786,9 @@ 5.32.0 6.29.0 5.23.0 + + 2.0.0.AM26 + 2.0.0 From 7133cdd62c5853ac707921a8e3f5bec2a0e1d0db Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Wed, 21 May 2025 11:32:50 +0530 Subject: [PATCH 2/2] Fix failure in precommit checks for Yetus JDK8 Hadoop2 due to missing class SchemaLdifExtractor in ApacheDS 2.0.0.AM26. The issue arises from an overridden ApacheDS version incompatible with Hadoop 2. To resolve, skip LDAP tests for Hadoop 2 and apply dependency changes only for Hadoop 3. --- hbase-http/pom.xml | 127 ++++++++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 60 deletions(-) diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 7a8f37521fb1..4637aae9afc4 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -147,60 +147,6 @@ log4j-1.2-api test - - org.apache.directory.server - apacheds-core - ${apacheds.version} - test - - - org.bouncycastle - bcprov-jdk15on - - - - - org.apache.directory.server - apacheds-protocol-ldap - ${apacheds.version} - test - - - org.bouncycastle - bcprov-jdk15on - - - - - org.apache.directory.server - apacheds-ldif-partition - ${apacheds.version} - test - - - org.apache.directory.api - api-ldap-codec-core - ${ldap-api.version} - test - - - org.apache.directory.api - api-ldap-model - ${ldap-api.version} - test - - - org.apache.directory.server - apacheds-server-integ - ${apacheds.version} - test - - - log4j - log4j - - - @@ -411,12 +357,6 @@ org.apache.hadoop hadoop-minikdc test - - - org.apache.directory.api - api-all - - org.apache.kerby @@ -445,6 +385,19 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + + **/org/apache/hadoop/hbase/http/TestLdapHttpServer** + + + @@ -493,6 +446,60 @@ hadoop-minikdc test + + org.apache.directory.server + apacheds-core + ${apacheds.version} + test + + + org.bouncycastle + bcprov-jdk15on + + + + + org.apache.directory.server + apacheds-protocol-ldap + ${apacheds.version} + test + + + org.bouncycastle + bcprov-jdk15on + + + + + org.apache.directory.server + apacheds-ldif-partition + ${apacheds.version} + test + + + org.apache.directory.api + api-ldap-codec-core + ${ldap-api.version} + test + + + org.apache.directory.api + api-ldap-model + ${ldap-api.version} + test + + + org.apache.directory.server + apacheds-server-integ + ${apacheds.version} + test + + + log4j + log4j + + +