From 5e6fcfd987dff05cdc1016675a0941080bb55220 Mon Sep 17 00:00:00 2001 From: "yoshida.shinya" Date: Tue, 17 Aug 2021 18:09:17 +0900 Subject: [PATCH 1/3] HBASE-26205 Use specified cluster conf for UserProvider in TableMRUtil#initCredentialsForCluster --- .../org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 9b8bec631420..8ba2c5e983fa 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -585,7 +585,7 @@ public static void initCredentialsForCluster(Job job, String quorumAddress) */ public static void initCredentialsForCluster(Job job, Configuration conf) throws IOException { - UserProvider userProvider = UserProvider.instantiate(job.getConfiguration()); + UserProvider userProvider = UserProvider.instantiate(conf); if (userProvider.isHBaseSecurityEnabled()) { try { Connection peerConn = ConnectionFactory.createConnection(conf); From b77ba667fbcf15dc7645b6407a7630bf42498079 Mon Sep 17 00:00:00 2001 From: "yoshida.shinya" Date: Wed, 18 Aug 2021 21:35:17 +0900 Subject: [PATCH 2/3] Add testcases --- hbase-mapreduce/pom.xml | 5 + .../mapreduce/TestTableMapReduceUtil.java | 179 +++++++++++++++++- 2 files changed, 179 insertions(+), 5 deletions(-) diff --git a/hbase-mapreduce/pom.xml b/hbase-mapreduce/pom.xml index 571e96e16e9d..92db1980b660 100644 --- a/hbase-mapreduce/pom.xml +++ b/hbase-mapreduce/pom.xml @@ -226,6 +226,11 @@ test-jar test + + org.apache.hadoop + hadoop-minikdc + test + org.mockito mockito-core diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java index 54415dfbe479..d64ca3a84ab4 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java @@ -17,18 +17,38 @@ */ package org.apache.hadoop.hbase.mapreduce; +import static org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; - +import static org.junit.Assert.assertTrue; +import java.io.File; +import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.access.AccessController; +import org.apache.hadoop.hbase.security.access.PermissionStorage; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; +import org.apache.hadoop.hbase.security.provider.SaslClientAuthenticationProviders; +import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; +import org.apache.hadoop.hbase.security.token.TokenProvider; +import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.junit.After; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -38,11 +58,17 @@ */ @Category({MapReduceTests.class, MediumTests.class}) public class TestTableMapReduceUtil { + private static final String HTTP_PRINCIPAL = "HTTP/localhost"; @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestTableMapReduceUtil.class); + @After + public void after() { + SaslClientAuthenticationProviders.reset(); + } + /* * initTableSnapshotMapperJob is tested in {@link TestTableSnapshotInputFormat} because * the method depends on an online cluster. @@ -51,7 +77,7 @@ public class TestTableMapReduceUtil { @Test public void testInitTableMapperJob1() throws Exception { Configuration configuration = new Configuration(); - Job job = new Job(configuration, "tableName"); + Job job = Job.getInstance(configuration, "tableName"); // test TableMapReduceUtil.initTableMapperJob("Table", new Scan(), Import.Importer.class, Text.class, Text.class, job, false, WALInputFormat.class); @@ -66,7 +92,7 @@ public void testInitTableMapperJob1() throws Exception { @Test public void testInitTableMapperJob2() throws Exception { Configuration configuration = new Configuration(); - Job job = new Job(configuration, "tableName"); + Job job = Job.getInstance(configuration, "tableName"); TableMapReduceUtil.initTableMapperJob(Bytes.toBytes("Table"), new Scan(), Import.Importer.class, Text.class, Text.class, job, false, WALInputFormat.class); assertEquals(WALInputFormat.class, job.getInputFormatClass()); @@ -80,7 +106,7 @@ public void testInitTableMapperJob2() throws Exception { @Test public void testInitTableMapperJob3() throws Exception { Configuration configuration = new Configuration(); - Job job = new Job(configuration, "tableName"); + Job job = Job.getInstance(configuration, "tableName"); TableMapReduceUtil.initTableMapperJob(Bytes.toBytes("Table"), new Scan(), Import.Importer.class, Text.class, Text.class, job); assertEquals(TableInputFormat.class, job.getInputFormatClass()); @@ -94,7 +120,7 @@ public void testInitTableMapperJob3() throws Exception { @Test public void testInitTableMapperJob4() throws Exception { Configuration configuration = new Configuration(); - Job job = new Job(configuration, "tableName"); + Job job = Job.getInstance(configuration, "tableName"); TableMapReduceUtil.initTableMapperJob(Bytes.toBytes("Table"), new Scan(), Import.Importer.class, Text.class, Text.class, job, false); assertEquals(TableInputFormat.class, job.getInputFormatClass()); @@ -104,4 +130,147 @@ public void testInitTableMapperJob4() throws Exception { assertNull(job.getCombinerClass()); assertEquals("Table", job.getConfiguration().get(TableInputFormat.INPUT_TABLE)); } + + private static void startSecureMiniCluster(HBaseTestingUtil util, MiniKdc kdc, String principal) + throws Exception { + Configuration conf = util.getConfiguration(); + + SecureTestUtil.enableSecurity(conf); + VisibilityTestUtil.enableVisiblityLabels(conf); + SecureTestUtil.verifyConfiguration(conf); + + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + AccessController.class.getName() + ',' + TokenProvider.class.getName()); + + HBaseKerberosUtils.setSecuredConfiguration(conf, + principal + '@' + kdc.getRealm(), HTTP_PRINCIPAL + '@' + kdc.getRealm()); + + util.startMiniCluster(); + util.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); + } + + @Test + public void testInitCredentialsForCluster1() throws Exception { + HBaseTestingUtil util1 = new HBaseTestingUtil(); + HBaseTestingUtil util2 = new HBaseTestingUtil(); + + util1.startMiniCluster(); + util2.startMiniCluster(); + + Configuration conf1 = util1.getConfiguration(); + Job job = Job.getInstance(conf1); + + TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); + + Credentials credentials = job.getCredentials(); + Collection> tokens = credentials.getAllTokens(); + assertTrue(tokens.isEmpty()); + + util1.shutdownMiniCluster(); + util2.shutdownMiniCluster(); + } + + @Test + @SuppressWarnings("unchecked") + public void testInitCredentialsForCluster2() throws Exception { + HBaseTestingUtil util1 = new HBaseTestingUtil(); + File keytab = new File(util1.getDataTestDir("keytab").toUri().getPath()); + MiniKdc kdc = util1.setupMiniKdc(keytab); + + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + startSecureMiniCluster(util1, kdc, userPrincipal); + + HBaseTestingUtil util2 = new HBaseTestingUtil(); + startSecureMiniCluster(util2, kdc, userPrincipal); + + Configuration conf1 = util1.getConfiguration(); + Job job = Job.getInstance(conf1); + + + TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); + + Credentials credentials = job.getCredentials(); + Collection> tokens = credentials.getAllTokens(); + assertEquals(1, tokens.size()); + + String clusterId = ZKClusterId.readClusterIdZNode(util2.getZooKeeperWatcher()); + Token tokenForCluster = + (Token) credentials.getToken(new Text(clusterId)); + assertEquals(userPrincipal + '@' + kdc.getRealm(), + tokenForCluster.decodeIdentifier().getUsername()); + + util1.shutdownMiniCluster(); + util2.shutdownMiniCluster(); + } + + @Test + public void testInitCredentialsForCluster3() throws Exception { + HBaseTestingUtil util1 = new HBaseTestingUtil(); + File keytab = new File(util1.getDataTestDir("keytab").toUri().getPath()); + MiniKdc kdc = util1.setupMiniKdc(keytab); + + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + startSecureMiniCluster(util1, kdc, userPrincipal); + + HBaseTestingUtil util2 = new HBaseTestingUtil(); + // Assume util2 is insecure cluster + // Do not start util2 because cannot boot secured mini cluster and insecure mini cluster at once + + Configuration conf1 = util1.getConfiguration(); + Job job = Job.getInstance(conf1); + + TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); + + Credentials credentials = job.getCredentials(); + Collection> tokens = credentials.getAllTokens(); + assertTrue(tokens.isEmpty()); + + + util1.shutdownMiniCluster(); + } + + @Test + @SuppressWarnings("unchecked") + public void testInitCredentialsForCluster4() throws Exception { + HBaseTestingUtil util1 = new HBaseTestingUtil(); + // Assume util1 is insecure cluster + // Do not start util1 because cannot boot secured mini cluster and insecure mini cluster at once + + HBaseTestingUtil util2 = new HBaseTestingUtil(); + + File keytab = new File(util2.getDataTestDir("keytab").toUri().getPath()); + MiniKdc kdc = util2.setupMiniKdc(keytab); + + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + startSecureMiniCluster(util2, kdc, userPrincipal); + + Configuration conf1 = util1.getConfiguration(); + Job job = Job.getInstance(conf1); + + TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); + + Credentials credentials = job.getCredentials(); + Collection> tokens = credentials.getAllTokens(); + assertEquals(1, tokens.size()); + + String clusterId = ZKClusterId.readClusterIdZNode(util2.getZooKeeperWatcher()); + Token tokenForCluster = + (Token) credentials.getToken(new Text(clusterId)); + assertEquals(userPrincipal + '@' + kdc.getRealm(), + tokenForCluster.decodeIdentifier().getUsername()); + + util2.shutdownMiniCluster(); + } } From 766fb9cb5301012ab6eb952a8eb615431be365dd Mon Sep 17 00:00:00 2001 From: "yoshida.shinya" Date: Thu, 19 Aug 2021 19:12:05 +0900 Subject: [PATCH 3/3] Better resource management for mini clusters --- .../mapreduce/TestTableMapReduceUtil.java | 235 +++++++++--------- 1 file changed, 124 insertions(+), 111 deletions(-) diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java index d64ca3a84ab4..09cdc279bc8a 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.io.Closeable; import java.io.File; import java.util.Collection; import org.apache.hadoop.conf.Configuration; @@ -79,8 +80,9 @@ public void testInitTableMapperJob1() throws Exception { Configuration configuration = new Configuration(); Job job = Job.getInstance(configuration, "tableName"); // test - TableMapReduceUtil.initTableMapperJob("Table", new Scan(), Import.Importer.class, Text.class, - Text.class, job, false, WALInputFormat.class); + TableMapReduceUtil.initTableMapperJob( + "Table", new Scan(), Import.Importer.class, Text.class, Text.class, job, + false, WALInputFormat.class); assertEquals(WALInputFormat.class, job.getInputFormatClass()); assertEquals(Import.Importer.class, job.getMapperClass()); assertEquals(LongWritable.class, job.getOutputKeyClass()); @@ -93,8 +95,9 @@ public void testInitTableMapperJob1() throws Exception { public void testInitTableMapperJob2() throws Exception { Configuration configuration = new Configuration(); Job job = Job.getInstance(configuration, "tableName"); - TableMapReduceUtil.initTableMapperJob(Bytes.toBytes("Table"), new Scan(), - Import.Importer.class, Text.class, Text.class, job, false, WALInputFormat.class); + TableMapReduceUtil.initTableMapperJob( + Bytes.toBytes("Table"), new Scan(), Import.Importer.class, Text.class, + Text.class, job, false, WALInputFormat.class); assertEquals(WALInputFormat.class, job.getInputFormatClass()); assertEquals(Import.Importer.class, job.getMapperClass()); assertEquals(LongWritable.class, job.getOutputKeyClass()); @@ -107,8 +110,9 @@ public void testInitTableMapperJob2() throws Exception { public void testInitTableMapperJob3() throws Exception { Configuration configuration = new Configuration(); Job job = Job.getInstance(configuration, "tableName"); - TableMapReduceUtil.initTableMapperJob(Bytes.toBytes("Table"), new Scan(), - Import.Importer.class, Text.class, Text.class, job); + TableMapReduceUtil.initTableMapperJob( + Bytes.toBytes("Table"), new Scan(), Import.Importer.class, Text.class, + Text.class, job); assertEquals(TableInputFormat.class, job.getInputFormatClass()); assertEquals(Import.Importer.class, job.getMapperClass()); assertEquals(LongWritable.class, job.getOutputKeyClass()); @@ -121,8 +125,9 @@ public void testInitTableMapperJob3() throws Exception { public void testInitTableMapperJob4() throws Exception { Configuration configuration = new Configuration(); Job job = Job.getInstance(configuration, "tableName"); - TableMapReduceUtil.initTableMapperJob(Bytes.toBytes("Table"), new Scan(), - Import.Importer.class, Text.class, Text.class, job, false); + TableMapReduceUtil.initTableMapperJob( + Bytes.toBytes("Table"), new Scan(), Import.Importer.class, Text.class, + Text.class, job, false); assertEquals(TableInputFormat.class, job.getInputFormatClass()); assertEquals(Import.Importer.class, job.getMapperClass()); assertEquals(LongWritable.class, job.getOutputKeyClass()); @@ -131,8 +136,8 @@ public void testInitTableMapperJob4() throws Exception { assertEquals("Table", job.getConfiguration().get(TableInputFormat.INPUT_TABLE)); } - private static void startSecureMiniCluster(HBaseTestingUtil util, MiniKdc kdc, String principal) - throws Exception { + private static Closeable startSecureMiniCluster( + HBaseTestingUtil util, MiniKdc kdc, String principal) throws Exception { Configuration conf = util.getConfiguration(); SecureTestUtil.enableSecurity(conf); @@ -146,131 +151,139 @@ private static void startSecureMiniCluster(HBaseTestingUtil util, MiniKdc kdc, S principal + '@' + kdc.getRealm(), HTTP_PRINCIPAL + '@' + kdc.getRealm()); util.startMiniCluster(); - util.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); + try { + util.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); + } catch (Exception e) { + util.shutdownMiniCluster(); + throw e; + } + + return util::shutdownMiniCluster; } - @Test - public void testInitCredentialsForCluster1() throws Exception { + @Test public void testInitCredentialsForCluster1() throws Exception { HBaseTestingUtil util1 = new HBaseTestingUtil(); HBaseTestingUtil util2 = new HBaseTestingUtil(); util1.startMiniCluster(); - util2.startMiniCluster(); - - Configuration conf1 = util1.getConfiguration(); - Job job = Job.getInstance(conf1); - - TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); - - Credentials credentials = job.getCredentials(); - Collection> tokens = credentials.getAllTokens(); - assertTrue(tokens.isEmpty()); - - util1.shutdownMiniCluster(); - util2.shutdownMiniCluster(); + try { + util2.startMiniCluster(); + try { + Configuration conf1 = util1.getConfiguration(); + Job job = Job.getInstance(conf1); + + TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); + + Credentials credentials = job.getCredentials(); + Collection> tokens = credentials.getAllTokens(); + assertTrue(tokens.isEmpty()); + } finally { + util2.shutdownMiniCluster(); + } + } finally { + util1.shutdownMiniCluster(); + } } - @Test - @SuppressWarnings("unchecked") - public void testInitCredentialsForCluster2() throws Exception { + @Test @SuppressWarnings("unchecked") public void testInitCredentialsForCluster2() + throws Exception { HBaseTestingUtil util1 = new HBaseTestingUtil(); - File keytab = new File(util1.getDataTestDir("keytab").toUri().getPath()); - MiniKdc kdc = util1.setupMiniKdc(keytab); - - String username = UserGroupInformation.getLoginUser().getShortUserName(); - String userPrincipal = username + "/localhost"; - kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); - loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - - startSecureMiniCluster(util1, kdc, userPrincipal); - HBaseTestingUtil util2 = new HBaseTestingUtil(); - startSecureMiniCluster(util2, kdc, userPrincipal); - - Configuration conf1 = util1.getConfiguration(); - Job job = Job.getInstance(conf1); - - - TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); - Credentials credentials = job.getCredentials(); - Collection> tokens = credentials.getAllTokens(); - assertEquals(1, tokens.size()); - - String clusterId = ZKClusterId.readClusterIdZNode(util2.getZooKeeperWatcher()); - Token tokenForCluster = - (Token) credentials.getToken(new Text(clusterId)); - assertEquals(userPrincipal + '@' + kdc.getRealm(), - tokenForCluster.decodeIdentifier().getUsername()); - - util1.shutdownMiniCluster(); - util2.shutdownMiniCluster(); + File keytab = new File(util1.getDataTestDir("keytab").toUri().getPath()); + MiniKdc kdc = util1.setupMiniKdc(keytab); + try { + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal); + Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { + Configuration conf1 = util1.getConfiguration(); + Job job = Job.getInstance(conf1); + + TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); + + Credentials credentials = job.getCredentials(); + Collection> tokens = credentials.getAllTokens(); + assertEquals(1, tokens.size()); + + String clusterId = ZKClusterId.readClusterIdZNode(util2.getZooKeeperWatcher()); + Token tokenForCluster = + (Token) credentials.getToken(new Text(clusterId)); + assertEquals(userPrincipal + '@' + kdc.getRealm(), + tokenForCluster.decodeIdentifier().getUsername()); + } + } finally { + kdc.stop(); + } } - @Test - public void testInitCredentialsForCluster3() throws Exception { + @Test public void testInitCredentialsForCluster3() throws Exception { HBaseTestingUtil util1 = new HBaseTestingUtil(); + File keytab = new File(util1.getDataTestDir("keytab").toUri().getPath()); MiniKdc kdc = util1.setupMiniKdc(keytab); - - String username = UserGroupInformation.getLoginUser().getShortUserName(); - String userPrincipal = username + "/localhost"; - kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); - loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - - startSecureMiniCluster(util1, kdc, userPrincipal); - - HBaseTestingUtil util2 = new HBaseTestingUtil(); - // Assume util2 is insecure cluster - // Do not start util2 because cannot boot secured mini cluster and insecure mini cluster at once - - Configuration conf1 = util1.getConfiguration(); - Job job = Job.getInstance(conf1); - - TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); - - Credentials credentials = job.getCredentials(); - Collection> tokens = credentials.getAllTokens(); - assertTrue(tokens.isEmpty()); - - - util1.shutdownMiniCluster(); + try { + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal)) { + HBaseTestingUtil util2 = new HBaseTestingUtil(); + // Assume util2 is insecure cluster + // Do not start util2 because cannot boot secured mini cluster and insecure mini cluster at + // once + + Configuration conf1 = util1.getConfiguration(); + Job job = Job.getInstance(conf1); + + TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); + + Credentials credentials = job.getCredentials(); + Collection> tokens = credentials.getAllTokens(); + assertTrue(tokens.isEmpty()); + } + } finally { + kdc.stop(); + } } - @Test - @SuppressWarnings("unchecked") - public void testInitCredentialsForCluster4() throws Exception { + @Test @SuppressWarnings("unchecked") public void testInitCredentialsForCluster4() + throws Exception { HBaseTestingUtil util1 = new HBaseTestingUtil(); // Assume util1 is insecure cluster // Do not start util1 because cannot boot secured mini cluster and insecure mini cluster at once HBaseTestingUtil util2 = new HBaseTestingUtil(); - File keytab = new File(util2.getDataTestDir("keytab").toUri().getPath()); MiniKdc kdc = util2.setupMiniKdc(keytab); - - String username = UserGroupInformation.getLoginUser().getShortUserName(); - String userPrincipal = username + "/localhost"; - kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); - loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - - startSecureMiniCluster(util2, kdc, userPrincipal); - - Configuration conf1 = util1.getConfiguration(); - Job job = Job.getInstance(conf1); - - TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); - - Credentials credentials = job.getCredentials(); - Collection> tokens = credentials.getAllTokens(); - assertEquals(1, tokens.size()); - - String clusterId = ZKClusterId.readClusterIdZNode(util2.getZooKeeperWatcher()); - Token tokenForCluster = - (Token) credentials.getToken(new Text(clusterId)); - assertEquals(userPrincipal + '@' + kdc.getRealm(), - tokenForCluster.decodeIdentifier().getUsername()); - - util2.shutdownMiniCluster(); + try { + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + try (Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { + Configuration conf1 = util1.getConfiguration(); + Job job = Job.getInstance(conf1); + + TableMapReduceUtil.initCredentialsForCluster(job, util2.getConfiguration()); + + Credentials credentials = job.getCredentials(); + Collection> tokens = credentials.getAllTokens(); + assertEquals(1, tokens.size()); + + String clusterId = ZKClusterId.readClusterIdZNode(util2.getZooKeeperWatcher()); + Token tokenForCluster = + (Token) credentials.getToken(new Text(clusterId)); + assertEquals(userPrincipal + '@' + kdc.getRealm(), + tokenForCluster.decodeIdentifier().getUsername()); + } + } finally { + kdc.stop(); + } } }