From c83e7894bc33fa61bb9ea6317ccf6307e8a167ba Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 30 Jul 2020 00:14:06 +0530 Subject: [PATCH 1/5] HBASE-24795 : RegionMover to deal with unknown region while (un)loading --- .../apache/hadoop/hbase/util/MoveWithAck.java | 153 +++++++++++++++ .../hadoop/hbase/util/MoveWithoutAck.java | 70 +++++++ .../apache/hadoop/hbase/util/RegionMover.java | 180 ++---------------- .../hadoop/hbase/util/TestRegionMover.java | 47 ++++- 4 files changed, 282 insertions(+), 168 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithoutAck.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java new file mode 100644 index 000000000000..be9db6db757e --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java @@ -0,0 +1,153 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.Callable; + +/** + * Move Regions and make sure that they are up on the target server.If a region movement fails we + * exit as failure + */ +@InterfaceAudience.Private +class MoveWithAck implements Callable { + + private static final Logger LOG = LoggerFactory.getLogger(MoveWithAck.class); + + private final RegionInfo region; + private final ServerName targetServer; + private final List movedRegions; + private final ServerName sourceServer; + private final Connection conn; + private final Admin admin; + + MoveWithAck(Connection conn, RegionInfo regionInfo, ServerName sourceServer, + ServerName targetServer, List movedRegions) throws IOException { + this.conn = conn; + this.region = regionInfo; + this.targetServer = targetServer; + this.movedRegions = movedRegions; + this.sourceServer = sourceServer; + this.admin = conn.getAdmin(); + } + + @Override + public Boolean call() throws IOException, InterruptedException { + boolean moved = false; + int count = 0; + int retries = admin.getConfiguration() + .getInt(RegionMover.MOVE_RETRIES_MAX_KEY, RegionMover.DEFAULT_MOVE_RETRIES_MAX); + int maxWaitInSeconds = admin.getConfiguration() + .getInt(RegionMover.MOVE_WAIT_MAX_KEY, RegionMover.DEFAULT_MOVE_WAIT_MAX); + long startTime = EnvironmentEdgeManager.currentTime(); + boolean sameServer = true; + // Assert we can scan the region in its current location + isSuccessfulScan(region); + LOG + .info("Moving region: {} from {} to {}", region.getEncodedName(), sourceServer, targetServer); + while (count < retries && sameServer) { + if (count > 0) { + LOG.info("Retry " + count + " of maximum " + retries); + } + count = count + 1; + admin.move(region.getEncodedNameAsBytes(), targetServer); + long maxWait = startTime + (maxWaitInSeconds * 1000); + while (EnvironmentEdgeManager.currentTime() < maxWait) { + sameServer = isSameServer(region, sourceServer); + if (!sameServer) { + break; + } + Thread.sleep(100); + } + } + if (sameServer) { + LOG.error("Region: {} stuck on {} ,newServer={}", region.getRegionNameAsString(), + this.sourceServer, this.targetServer); + } else { + isSuccessfulScan(region); + LOG.info("Moved Region " + region.getRegionNameAsString() + " cost:" + String + .format("%.3f", (float) (EnvironmentEdgeManager.currentTime() - startTime) / 1000)); + moved = true; + movedRegions.add(region); + } + return moved; + } + + /** + * Tries to scan a row from passed region + */ + private void isSuccessfulScan(RegionInfo region) throws IOException { + Scan scan = new Scan().withStartRow(region.getStartKey()).setRaw(true).setOneRowLimit() + .setMaxResultSize(1L).setCaching(1).setFilter(new FirstKeyOnlyFilter()) + .setCacheBlocks(false); + try (Table table = conn.getTable(region.getTable()); + ResultScanner scanner = table.getScanner(scan)) { + scanner.next(); + } catch (IOException e) { + LOG.error("Could not scan region:" + region.getEncodedName(), e); + throw e; + } + } + + /** + * Returns true if passed region is still on serverName when we look at hbase:meta. + * @return true if region is hosted on serverName otherwise false + */ + private boolean isSameServer(RegionInfo region, ServerName serverName) + throws IOException { + ServerName serverForRegion = getServerNameForRegion(region, admin, conn); + return serverForRegion != null && serverForRegion.equals(serverName); + } + + /** + * Get servername that is up in hbase:meta hosting the given region. this is hostname + port + + * startcode comma-delimited. Can return null + * @return regionServer hosting the given region + */ + static ServerName getServerNameForRegion(RegionInfo region, Admin admin, Connection conn) + throws IOException { + if (!admin.isTableEnabled(region.getTable())) { + return null; + } + HRegionLocation loc = + conn.getRegionLocator(region.getTable()).getRegionLocation(region.getStartKey(), + region.getReplicaId(),true); + if (loc != null) { + return loc.getServerName(); + } else { + return null; + } + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithoutAck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithoutAck.java new file mode 100644 index 000000000000..e111302daae7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithoutAck.java @@ -0,0 +1,70 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.Callable; + +/** + * Move Regions without Acknowledging.Usefule in case of RS shutdown as we might want to shut the + * RS down anyways and not abort on a stuck region. Improves movement performance + */ +@InterfaceAudience.Private +class MoveWithoutAck implements Callable { + + private static final Logger LOG = LoggerFactory.getLogger(MoveWithoutAck.class); + + private final RegionInfo region; + private final ServerName targetServer; + private final List movedRegions; + private final ServerName sourceServer; + private final Admin admin; + + MoveWithoutAck(Admin admin, RegionInfo regionInfo, ServerName sourceServer, + ServerName targetServer, List movedRegions) { + this.admin = admin; + this.region = regionInfo; + this.targetServer = targetServer; + this.movedRegions = movedRegions; + this.sourceServer = sourceServer; + } + + @Override + public Boolean call() { + try { + LOG.info("Moving region: {} from {} to {}", region.getEncodedName(), sourceServer, + targetServer); + admin.move(region.getEncodedNameAsBytes(), targetServer); + LOG.info("Moved {} from {} to {}", region.getEncodedName(), sourceServer, targetServer); + } catch (Exception e) { + LOG.error("Error Moving Region: {}", region.getEncodedName(), e); + } finally { + movedRegions.add(region); + } + return true; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 693bfc5adda2..b7119f1db23c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -39,7 +39,6 @@ import java.util.List; import java.util.Locale; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -53,16 +52,12 @@ import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -259,105 +254,6 @@ public RegionMover build() throws IOException { } } - /** - * Move Regions and make sure that they are up on the target server.If a region movement fails we - * exit as failure - */ - private class MoveWithAck implements Callable { - private RegionInfo region; - private ServerName targetServer; - private List movedRegions; - private ServerName sourceServer; - - public MoveWithAck(RegionInfo regionInfo, ServerName sourceServer, - ServerName targetServer, List movedRegions) { - this.region = regionInfo; - this.targetServer = targetServer; - this.movedRegions = movedRegions; - this.sourceServer = sourceServer; - } - - @Override - public Boolean call() throws IOException, InterruptedException { - boolean moved = false; - int count = 0; - int retries = admin.getConfiguration().getInt(MOVE_RETRIES_MAX_KEY, DEFAULT_MOVE_RETRIES_MAX); - int maxWaitInSeconds = - admin.getConfiguration().getInt(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); - long startTime = EnvironmentEdgeManager.currentTime(); - boolean sameServer = true; - // Assert we can scan the region in its current location - isSuccessfulScan(region); - LOG.info("Moving region:" + region.getEncodedName() + " from " + sourceServer + " to " - + targetServer); - while (count < retries && sameServer) { - if (count > 0) { - LOG.info("Retry " + Integer.toString(count) + " of maximum " + Integer.toString(retries)); - } - count = count + 1; - admin.move(region.getEncodedNameAsBytes(), targetServer); - long maxWait = startTime + (maxWaitInSeconds * 1000); - while (EnvironmentEdgeManager.currentTime() < maxWait) { - sameServer = isSameServer(region, sourceServer); - if (!sameServer) { - break; - } - Thread.sleep(100); - } - } - if (sameServer) { - LOG.error("Region: " + region.getRegionNameAsString() + " stuck on " + this.sourceServer - + ",newServer=" + this.targetServer); - } else { - isSuccessfulScan(region); - LOG.info("Moved Region " - + region.getRegionNameAsString() - + " cost:" - + String.format("%.3f", - (float) (EnvironmentEdgeManager.currentTime() - startTime) / 1000)); - moved = true; - movedRegions.add(region); - } - return moved; - } - } - - /** - * Move Regions without Acknowledging.Usefule in case of RS shutdown as we might want to shut the - * RS down anyways and not abort on a stuck region. Improves movement performance - */ - private class MoveWithoutAck implements Callable { - private RegionInfo region; - private ServerName targetServer; - private List movedRegions; - private ServerName sourceServer; - - public MoveWithoutAck(RegionInfo regionInfo, ServerName sourceServer, - ServerName targetServer, List movedRegions) { - this.region = regionInfo; - this.targetServer = targetServer; - this.movedRegions = movedRegions; - this.sourceServer = sourceServer; - } - - @Override - public Boolean call() { - try { - LOG.info("Moving region:" + region.getEncodedName() + " from " + sourceServer + " to " - + targetServer); - admin.move(region.getEncodedNameAsBytes(), targetServer); - LOG.info("Moved " + region.getEncodedName() + " from " + sourceServer + " to " - + targetServer); - } catch (Exception e) { - LOG.error("Error Moving Region:" + region.getEncodedName(), e); - } finally { - // we add region to the moved regions list in No Ack Mode since this is best effort - movedRegions.add(region); - } - return true; - } - } - /** * Loads the specified {@link #hostname} with regions listed in the {@link #filename} RegionMover * Object has to be created using {@link #RegionMover(RegionMoverBuilder)} @@ -395,7 +291,7 @@ private void loadRegions(List regionsToMove) int counter = 0; while (counter < regionsToMove.size()) { RegionInfo region = regionsToMove.get(counter); - ServerName currentServer = getServerNameForRegion(region); + ServerName currentServer = MoveWithAck.getServerNameForRegion(region, admin, conn); if (currentServer == null) { LOG.warn( "Could not get server for Region:" + region.getRegionNameAsString() + " moving on"); @@ -408,12 +304,12 @@ private void loadRegions(List regionsToMove) continue; } if (ack) { - Future task = - moveRegionsPool.submit(new MoveWithAck(region, currentServer, server, movedRegions)); + Future task = moveRegionsPool + .submit(new MoveWithAck(conn, region, currentServer, server, movedRegions)); taskList.add(task); } else { - Future task = - moveRegionsPool.submit(new MoveWithoutAck(region, currentServer, server, movedRegions)); + Future task = moveRegionsPool + .submit(new MoveWithoutAck(admin, region, currentServer, server, movedRegions)); taskList.add(task); } counter++; @@ -522,13 +418,13 @@ private void unloadRegions(ServerName server, List regionServers, while (counter < regionsToMove.size()) { if (ack) { Future task = moveRegionsPool.submit( - new MoveWithAck(regionsToMove.get(counter), server, regionServers.get(serverIndex), - movedRegions)); + new MoveWithAck(conn, regionsToMove.get(counter), server, + regionServers.get(serverIndex), movedRegions)); taskList.add(task); } else { Future task = moveRegionsPool.submit( - new MoveWithoutAck(regionsToMove.get(counter), server, regionServers.get(serverIndex), - movedRegions)); + new MoveWithoutAck(admin, regionsToMove.get(counter), server, + regionServers.get(serverIndex), movedRegions)); taskList.add(task); } counter++; @@ -587,8 +483,12 @@ private void waitMoveTasksToFinish(ExecutorService moveRegionsPool, LOG.error("Interrupted while waiting for Thread to Complete " + e.getMessage(), e); throw e; } catch (ExecutionException e) { - LOG.error("Got Exception From Thread While moving region " + e.getMessage(), e); - throw e; + if (e.getCause() instanceof UnknownRegionException) { + LOG.debug("Ignore unknown region, it might have been split/merged."); + } else { + LOG.error("Got Exception From Thread While moving region {}", e.getMessage(), e); + throw e; + } } catch (CancellationException e) { LOG.error("Thread for moving region cancelled. Timeout for cancellation:" + timeoutInSeconds + "secs", e); @@ -747,54 +647,6 @@ private ServerName stripServer(List regionServers, String hostname, return null; } - /** - * Tries to scan a row from passed region - */ - private void isSuccessfulScan(RegionInfo region) throws IOException { - Scan scan = new Scan().withStartRow(region.getStartKey()).setRaw(true).setOneRowLimit() - .setMaxResultSize(1L).setCaching(1).setFilter(new FirstKeyOnlyFilter()) - .setCacheBlocks(false); - try (Table table = conn.getTable(region.getTable()); - ResultScanner scanner = table.getScanner(scan)) { - scanner.next(); - } catch (IOException e) { - LOG.error("Could not scan region:" + region.getEncodedName(), e); - throw e; - } - } - - /** - * Returns true if passed region is still on serverName when we look at hbase:meta. - * @return true if region is hosted on serverName otherwise false - */ - private boolean isSameServer(RegionInfo region, ServerName serverName) - throws IOException { - ServerName serverForRegion = getServerNameForRegion(region); - if (serverForRegion != null && serverForRegion.equals(serverName)) { - return true; - } - return false; - } - - /** - * Get servername that is up in hbase:meta hosting the given region. this is hostname + port + - * startcode comma-delimited. Can return null - * @return regionServer hosting the given region - */ - private ServerName getServerNameForRegion(RegionInfo region) throws IOException { - if (!admin.isTableEnabled(region.getTable())) { - return null; - } - HRegionLocation loc = - conn.getRegionLocator(region.getTable()).getRegionLocation(region.getStartKey(), - region.getReplicaId(),true); - if (loc != null) { - return loc.getServerName(); - } else { - return null; - } - } - @Override protected void addOptions() { this.addRequiredOptWithArg("r", "regionserverhost", "region server |"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java index b7e947cca4db..f5ad8378ec3d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java @@ -23,8 +23,11 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -37,6 +40,8 @@ import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -69,6 +74,8 @@ public class TestRegionMover { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final TableName TABLE_NAME = TableName.valueOf("testRegionMover"); + @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(3); @@ -83,12 +90,11 @@ public static void tearDownAfterClass() throws Exception { @Before public void setUp() throws Exception { // Create a pre-split table just to populate some regions - TableName tableName = TableName.valueOf("testRegionMover"); Admin admin = TEST_UTIL.getAdmin(); - if (admin.tableExists(tableName)) { - TEST_UTIL.deleteTable(tableName); + if (admin.tableExists(TABLE_NAME)) { + TEST_UTIL.deleteTable(TABLE_NAME); } - TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(TABLE_NAME) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); String startKey = "a"; String endKey = "z"; @@ -457,4 +463,37 @@ public void testExcludeAndDecomServers() throws Exception { Collections.emptyList()); } + @Test + public void testWithMergedRegions() throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + Admin admin = TEST_UTIL.getAdmin(); + Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); + List puts = new ArrayList<>(); + for (int i = 0; i < 10000; i++) { + puts.add(new Put(Bytes.toBytes("rowkey_" + i)) + .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); + } + table.put(puts); + admin.flush(TABLE_NAME); + HRegionServer regionServer = cluster.getRegionServer(0); + String rsName = regionServer.getServerName().getAddress().toString(); + int numRegions = regionServer.getNumberOfOnlineRegions(); + List hRegions = regionServer.getRegions().stream() + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(TABLE_NAME)) + .collect(Collectors.toList()); + RegionMoverBuilder rmBuilder = + new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8); + try (RegionMover rm = rmBuilder.build()) { + LOG.debug("Unloading " + regionServer.getServerName()); + rm.unload(); + Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); + LOG.debug("Successfully Unloaded\nNow Loading"); + admin.mergeRegionsAsync(new byte[][] { hRegions.get(0).getRegionInfo().getRegionName(), + hRegions.get(1).getRegionInfo().getRegionName() }, true) + .get(5, TimeUnit.SECONDS); + Assert.assertTrue(rm.load()); + Assert.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions()); + } + } + } From 51a74d66b61d5ea9e2462284feb3b7b5c2f37958 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 30 Jul 2020 12:18:34 +0530 Subject: [PATCH 2/5] update based on review --- .../src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java | 2 +- .../src/main/java/org/apache/hadoop/hbase/util/RegionMover.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java index be9db6db757e..e6c317e5c24a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java @@ -88,7 +88,7 @@ public Boolean call() throws IOException, InterruptedException { if (!sameServer) { break; } - Thread.sleep(100); + Thread.sleep(1000); } } if (sameServer) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index b7119f1db23c..34ebe365687d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -484,7 +484,7 @@ private void waitMoveTasksToFinish(ExecutorService moveRegionsPool, throw e; } catch (ExecutionException e) { if (e.getCause() instanceof UnknownRegionException) { - LOG.debug("Ignore unknown region, it might have been split/merged."); + LOG.info("Ignore unknown region, it might have been split/merged."); } else { LOG.error("Got Exception From Thread While moving region {}", e.getMessage(), e); throw e; From bdec03c092fb18391fb30aca7b5fc72044275638 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Fri, 31 Jul 2020 04:18:29 +0530 Subject: [PATCH 3/5] update based on review --- .../master/assignment/AssignmentManager.java | 3 +- .../hadoop/hbase/util/MoveWithoutAck.java | 3 +- .../apache/hadoop/hbase/util/RegionMover.java | 24 ++- .../hadoop/hbase/util/TestRegionMover.java | 38 ---- .../hadoop/hbase/util/TestRegionMover2.java | 202 ++++++++++++++++++ 5 files changed, 228 insertions(+), 42 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 4c42237801bb..360f7d28ef34 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -150,6 +150,7 @@ public class AssignmentManager { public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD = "hbase.metrics.rit.stuck.warning.threshold"; private static final int DEFAULT_RIT_STUCK_WARNING_THRESHOLD = 60 * 1000; + public static final String UNEXPECTED_STATE_REGION = "Unexpected state for "; private final ProcedureEvent metaAssignEvent = new ProcedureEvent<>("meta assign"); private final ProcedureEvent metaLoadEvent = new ProcedureEvent<>("meta load"); @@ -588,7 +589,7 @@ private void preTransitCheck(RegionStateNode regionNode, RegionState.State[] exp throw new HBaseIOException(regionNode + " is currently in transition"); } if (!regionNode.isInState(expectedStates)) { - throw new DoNotRetryRegionException("Unexpected state for " + regionNode); + throw new DoNotRetryRegionException(UNEXPECTED_STATE_REGION + regionNode); } if (isTableDisabled(regionNode.getTable())) { throw new DoNotRetryIOException(regionNode.getTable() + " is disabled for " + regionNode); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithoutAck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithoutAck.java index e111302daae7..0ddb99ac4180 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithoutAck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithoutAck.java @@ -59,7 +59,8 @@ public Boolean call() { LOG.info("Moving region: {} from {} to {}", region.getEncodedName(), sourceServer, targetServer); admin.move(region.getEncodedNameAsBytes(), targetServer); - LOG.info("Moved {} from {} to {}", region.getEncodedName(), sourceServer, targetServer); + LOG.info("Requested move {} from {} to {}", region.getEncodedName(), sourceServer, + targetServer); } catch (Exception e) { LOG.error("Error Moving Region: {}", region.getEncodedName(), e); } finally { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 34ebe365687d..c8d1270f2871 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -57,7 +57,9 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.DoNotRetryRegionException; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -483,8 +485,10 @@ private void waitMoveTasksToFinish(ExecutorService moveRegionsPool, LOG.error("Interrupted while waiting for Thread to Complete " + e.getMessage(), e); throw e; } catch (ExecutionException e) { - if (e.getCause() instanceof UnknownRegionException) { - LOG.info("Ignore unknown region, it might have been split/merged."); + boolean ignoreFailure = ignoreRegionMoveFailure(e); + if (ignoreFailure) { + LOG.debug("Ignore region move failure, it might have been split/merged, " + + "detailed message: {}", e.getCause().getMessage()); } else { LOG.error("Got Exception From Thread While moving region {}", e.getMessage(), e); throw e; @@ -497,6 +501,22 @@ private void waitMoveTasksToFinish(ExecutorService moveRegionsPool, } } + private boolean ignoreRegionMoveFailure(ExecutionException e) { + boolean ignoreFailure; + if (e.getCause() instanceof UnknownRegionException) { + // region does not exist anymore + ignoreFailure = true; + } else if (e.getCause() instanceof DoNotRetryRegionException + && e.getCause().getMessage() != null && e.getCause().getMessage() + .contains(AssignmentManager.UNEXPECTED_STATE_REGION + "state=SPLIT,")) { + // region is recently split + ignoreFailure = true; + } else { + ignoreFailure = false; + } + return ignoreFailure; + } + private ServerName getTargetServer() throws Exception { ServerName server = null; int maxWaitInSeconds = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java index f5ad8378ec3d..87ce9ffd7339 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java @@ -23,11 +23,8 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -40,8 +37,6 @@ import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -463,37 +458,4 @@ public void testExcludeAndDecomServers() throws Exception { Collections.emptyList()); } - @Test - public void testWithMergedRegions() throws Exception { - MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); - Admin admin = TEST_UTIL.getAdmin(); - Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); - List puts = new ArrayList<>(); - for (int i = 0; i < 10000; i++) { - puts.add(new Put(Bytes.toBytes("rowkey_" + i)) - .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); - } - table.put(puts); - admin.flush(TABLE_NAME); - HRegionServer regionServer = cluster.getRegionServer(0); - String rsName = regionServer.getServerName().getAddress().toString(); - int numRegions = regionServer.getNumberOfOnlineRegions(); - List hRegions = regionServer.getRegions().stream() - .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(TABLE_NAME)) - .collect(Collectors.toList()); - RegionMoverBuilder rmBuilder = - new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8); - try (RegionMover rm = rmBuilder.build()) { - LOG.debug("Unloading " + regionServer.getServerName()); - rm.unload(); - Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); - LOG.debug("Successfully Unloaded\nNow Loading"); - admin.mergeRegionsAsync(new byte[][] { hRegions.get(0).getRegionInfo().getRegionName(), - hRegions.get(1).getRegionInfo().getRegionName() }, true) - .get(5, TimeUnit.SECONDS); - Assert.assertTrue(rm.load()); - Assert.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions()); - } - } - } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java new file mode 100644 index 000000000000..f73876556f4a --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test + * exclude functionality useful for rack decommissioning + */ +@Category({ MiscTests.class, LargeTests.class}) +public class TestRegionMover2 { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionMover2.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private static final TableName TABLE_NAME = TableName.valueOf("testRegionMover2"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(3); + TEST_UTIL.getAdmin().balancerSwitch(false, true); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + // Create a pre-split table just to populate some regions + Admin admin = TEST_UTIL.getAdmin(); + if (admin.tableExists(TABLE_NAME)) { + TEST_UTIL.deleteTable(TABLE_NAME); + } + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(TABLE_NAME) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); + int startKey = 0; + int endKey = 80000; + admin.createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); + } + + @Test + public void testWithMergedRegions() throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + Admin admin = TEST_UTIL.getAdmin(); + Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); + List puts = new ArrayList<>(); + for (int i = 0; i < 10000; i++) { + puts.add(new Put(Bytes.toBytes("rowkey_" + i)) + .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); + } + table.put(puts); + admin.flush(TABLE_NAME); + HRegionServer regionServer = cluster.getRegionServer(0); + String rsName = regionServer.getServerName().getAddress().toString(); + int numRegions = regionServer.getNumberOfOnlineRegions(); + List hRegions = regionServer.getRegions().stream() + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(TABLE_NAME)) + .collect(Collectors.toList()); + RegionMover.RegionMoverBuilder rmBuilder = + new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) + .maxthreads(8); + try (RegionMover rm = rmBuilder.build()) { + LOG.debug("Unloading " + regionServer.getServerName()); + rm.unload(); + Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); + LOG.debug("Successfully Unloaded\nNow Loading"); + admin.mergeRegionsAsync(new byte[][] { hRegions.get(0).getRegionInfo().getRegionName(), + hRegions.get(1).getRegionInfo().getRegionName() }, true) + .get(5, TimeUnit.SECONDS); + Assert.assertTrue(rm.load()); + Assert.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions()); + } + } + + @Test + public void testWithSplitRegions() throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + Admin admin = TEST_UTIL.getAdmin(); + Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); + List puts = new ArrayList<>(); + for (int i = 10; i < 50000; i++) { + puts.add(new Put(Bytes.toBytes(i)) + .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); + } + table.put(puts); + admin.flush(TABLE_NAME); + admin.compact(TABLE_NAME); + HRegionServer regionServer = cluster.getRegionServer(0); + String rsName = regionServer.getServerName().getAddress().toString(); + int numRegions = regionServer.getNumberOfOnlineRegions(); + List hRegions = regionServer.getRegions().stream() + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(TABLE_NAME)) + .collect(Collectors.toList()); + + RegionMover.RegionMoverBuilder rmBuilder = + new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) + .maxthreads(8); + try (RegionMover rm = rmBuilder.build()) { + LOG.debug("Unloading " + regionServer.getServerName()); + rm.unload(); + Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); + LOG.debug("Successfully Unloaded\nNow Loading"); + HRegion hRegion = hRegions.get(1); + int startKey = 0; + int endKey = Integer.MAX_VALUE; + if (hRegion.getRegionInfo().getStartKey().length > 0) { + startKey = Bytes.toInt(hRegion.getRegionInfo().getStartKey()); + } + if (hRegion.getRegionInfo().getEndKey().length > 0) { + endKey = Bytes.toInt(hRegion.getRegionInfo().getEndKey()); + } + int midKey = startKey + (endKey - startKey) / 2; + admin.splitRegionAsync(hRegion.getRegionInfo().getRegionName(), Bytes.toBytes(midKey)) + .get(5, TimeUnit.SECONDS); + Assert.assertTrue(rm.load()); + Assert.assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions()); + } + } + + @Test + public void testFailedRegionMove() throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + Admin admin = TEST_UTIL.getAdmin(); + Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); + List puts = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + puts.add(new Put(Bytes.toBytes("rowkey_" + i)) + .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); + } + table.put(puts); + admin.flush(TABLE_NAME); + HRegionServer regionServer = cluster.getRegionServer(0); + String rsName = regionServer.getServerName().getAddress().toString(); + int numRegions = regionServer.getNumberOfOnlineRegions(); + List hRegions = regionServer.getRegions().stream() + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(TABLE_NAME)) + .collect(Collectors.toList()); + RegionMover.RegionMoverBuilder rmBuilder = + new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) + .maxthreads(8); + try (RegionMover rm = rmBuilder.build()) { + LOG.debug("Unloading " + regionServer.getServerName()); + rm.unload(); + Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); + LOG.debug("Successfully Unloaded\nNow Loading"); + admin.offline(hRegions.get(0).getRegionInfo().getRegionName()); + // loading regions will fail because of offline region + Assert.assertFalse(rm.load()); + } + } + +} From 433cb91928aa4c800f8cdf3cf04a286c2fbb98a4 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Fri, 31 Jul 2020 17:06:03 +0530 Subject: [PATCH 4/5] addressing review --- .../apache/hadoop/hbase/util/MoveWithAck.java | 21 ++++-- .../apache/hadoop/hbase/util/RegionMover.java | 28 +++---- ...RegionMover.java => TestRegionMover1.java} | 73 ++++++++++++++----- .../hadoop/hbase/util/TestRegionMover2.java | 65 ++++++++++------- 4 files changed, 117 insertions(+), 70 deletions(-) rename hbase-server/src/test/java/org/apache/hadoop/hbase/util/{TestRegionMover.java => TestRegionMover1.java} (87%) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java index e6c317e5c24a..bde7fea1c366 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java @@ -74,11 +74,12 @@ public Boolean call() throws IOException, InterruptedException { boolean sameServer = true; // Assert we can scan the region in its current location isSuccessfulScan(region); - LOG - .info("Moving region: {} from {} to {}", region.getEncodedName(), sourceServer, targetServer); + LOG.info("Moving region: {} from {} to {}", region.getRegionNameAsString(), sourceServer, + targetServer); while (count < retries && sameServer) { if (count > 0) { - LOG.info("Retry " + count + " of maximum " + retries); + LOG.debug("Retry {} of maximum {} for region: {}", count, retries, + region.getRegionNameAsString()); } count = count + 1; admin.move(region.getEncodedNameAsBytes(), targetServer); @@ -92,18 +93,22 @@ public Boolean call() throws IOException, InterruptedException { } } if (sameServer) { - LOG.error("Region: {} stuck on {} ,newServer={}", region.getRegionNameAsString(), - this.sourceServer, this.targetServer); + LOG.error("Region: {} stuck on {} for {} sec , newServer={}", region.getRegionNameAsString(), + this.sourceServer, getTimeDiffInSec(startTime), this.targetServer); } else { isSuccessfulScan(region); - LOG.info("Moved Region " + region.getRegionNameAsString() + " cost:" + String - .format("%.3f", (float) (EnvironmentEdgeManager.currentTime() - startTime) / 1000)); + LOG.info("Moved Region {} , cost (sec): {}", region.getRegionNameAsString(), + getTimeDiffInSec(startTime)); moved = true; movedRegions.add(region); } return moved; } + private static String getTimeDiffInSec(long startTime) { + return String.format("%.3f", (float) (EnvironmentEdgeManager.currentTime() - startTime) / 1000); + } + /** * Tries to scan a row from passed region */ @@ -115,7 +120,7 @@ private void isSuccessfulScan(RegionInfo region) throws IOException { ResultScanner scanner = table.getScanner(scan)) { scanner.next(); } catch (IOException e) { - LOG.error("Could not scan region:" + region.getEncodedName(), e); + LOG.error("Could not scan region: {}", region.getEncodedName(), e); throw e; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index c8d1270f2871..03e674d54cc5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -288,7 +288,7 @@ private void loadRegions(List regionsToMove) "Moving " + regionsToMove.size() + " regions to " + server + " using " + this.maxthreads + " threads.Ack mode:" + this.ack); - ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); + final ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); List> taskList = new ArrayList<>(); int counter = 0; while (counter < regionsToMove.size()) { @@ -410,26 +410,23 @@ private void unloadRegions(ServerName server, List regionServers, LOG.info("No Regions to move....Quitting now"); break; } - int counter = 0; - LOG.info("Moving " + regionsToMove.size() + " regions from " + this.hostname + " to " - + regionServers.size() + " servers using " + this.maxthreads + " threads .Ack Mode:" - + ack); - ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); + LOG.info("Moving {} regions from {} to {} servers using {} threads .Ack Mode: {}", + regionsToMove.size(), this.hostname, regionServers.size(), this.maxthreads, ack); + final ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); List> taskList = new ArrayList<>(); int serverIndex = 0; - while (counter < regionsToMove.size()) { + for (RegionInfo regionToMove : regionsToMove) { if (ack) { Future task = moveRegionsPool.submit( - new MoveWithAck(conn, regionsToMove.get(counter), server, - regionServers.get(serverIndex), movedRegions)); + new MoveWithAck(conn, regionToMove, server, regionServers.get(serverIndex), + movedRegions)); taskList.add(task); } else { Future task = moveRegionsPool.submit( - new MoveWithoutAck(admin, regionsToMove.get(counter), server, - regionServers.get(serverIndex), movedRegions)); + new MoveWithoutAck(admin, regionToMove, server, regionServers.get(serverIndex), + movedRegions)); taskList.add(task); } - counter++; serverIndex = (serverIndex + 1) % regionServers.size(); } moveRegionsPool.shutdown(); @@ -487,8 +484,7 @@ private void waitMoveTasksToFinish(ExecutorService moveRegionsPool, } catch (ExecutionException e) { boolean ignoreFailure = ignoreRegionMoveFailure(e); if (ignoreFailure) { - LOG.debug("Ignore region move failure, it might have been split/merged, " - + "detailed message: {}", e.getCause().getMessage()); + LOG.debug("Ignore region move failure, it might have been split/merged.", e); } else { LOG.error("Got Exception From Thread While moving region {}", e.getMessage(), e); throw e; @@ -502,7 +498,7 @@ private void waitMoveTasksToFinish(ExecutorService moveRegionsPool, } private boolean ignoreRegionMoveFailure(ExecutionException e) { - boolean ignoreFailure; + boolean ignoreFailure = false; if (e.getCause() instanceof UnknownRegionException) { // region does not exist anymore ignoreFailure = true; @@ -511,8 +507,6 @@ private boolean ignoreRegionMoveFailure(ExecutionException e) { .contains(AssignmentManager.UNEXPECTED_STATE_REGION + "state=SPLIT,")) { // region is recently split ignoreFailure = true; - } else { - ignoreFailure = false; } return ignoreFailure; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover1.java similarity index 87% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover1.java index 87ce9ffd7339..a5b79ee92eb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover1.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter.Predicate; -import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; @@ -46,11 +45,12 @@ import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,17 +59,18 @@ * exclude functionality useful for rack decommissioning */ @Category({MiscTests.class, LargeTests.class}) -public class TestRegionMover { +public class TestRegionMover1 { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionMover.class); + HBaseClassTestRule.forClass(TestRegionMover1.class); - private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover.class); + @Rule + public TestName name = new TestName(); - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover1.class); - private static final TableName TABLE_NAME = TableName.valueOf("testRegionMover"); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -82,22 +83,18 @@ public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - @Before - public void setUp() throws Exception { - // Create a pre-split table just to populate some regions - Admin admin = TEST_UTIL.getAdmin(); - if (admin.tableExists(TABLE_NAME)) { - TEST_UTIL.deleteTable(TABLE_NAME); - } - TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(TABLE_NAME) + private void initTableRegions(TableName tableName) throws IOException { + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); String startKey = "a"; String endKey = "z"; - admin.createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); + TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); } @Test public void testWithAck() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getAddress().toString(); @@ -114,6 +111,8 @@ public void testWithAck() throws Exception { // Repeat the same load. It should be very fast because all regions are already moved. rm.load(); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } /** @@ -121,6 +120,8 @@ public void testWithAck() throws Exception { */ @Test public void testWithoutAck() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getAddress().toString(); @@ -147,6 +148,8 @@ public boolean evaluate() throws Exception { } }); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } /** @@ -155,6 +158,8 @@ public boolean evaluate() throws Exception { */ @Test public void testExclude() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); FileWriter fos = new FileWriter(excludeFile); @@ -179,10 +184,14 @@ public void testExclude() throws Exception { LOG.info("Before:" + regionsExcludeServer + " After:" + cluster.getRegionServer(1).getNumberOfOnlineRegions()); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testDesignatedFile() throws Exception{ + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "designated_file"); @@ -210,10 +219,14 @@ public void testDesignatedFile() throws Exception{ LOG.debug("Before:{} After:{}", regionsInDesignatedServer, designatedServer.getNumberOfOnlineRegions()); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testExcludeAndDesignated() throws Exception{ + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); // create designated file File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), @@ -258,10 +271,14 @@ public void testExcludeAndDesignated() throws Exception{ LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions()); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } @Test - public void testRegionServerPort() { + public void testRegionServerPort() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getHostname(); @@ -275,6 +292,8 @@ public void testRegionServerPort() { if (originalPort != null) { conf.set(HConstants.REGIONSERVER_PORT, originalPort); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } /** @@ -282,6 +301,8 @@ public void testRegionServerPort() { */ @Test public void testLoadMetaRegion() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() .map(t -> t.getRegionServer()) .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get(); @@ -296,6 +317,8 @@ public void testLoadMetaRegion() throws Exception { rm.load(); assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions()); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } /** @@ -303,6 +326,8 @@ public void testLoadMetaRegion() throws Exception { */ @Test public void testTargetServerDeadWhenLoading() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); String rsName = rs.getServerName().getAddress().toString(); Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); @@ -324,10 +349,14 @@ public void testTargetServerDeadWhenLoading() throws Exception { LOG.info("Loading to an inexist region server {}", inexistRsName); assertFalse(rm.load()); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testDecomServerExclusionWithAck() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer excludeServer = cluster.getRegionServer(1); List regions = excludeServer.getRegions(); @@ -367,6 +396,8 @@ public void testDecomServerExclusionWithAck() throws Exception { TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), Collections.emptyList()); + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } private void waitForServerDecom(HRegionServer excludeServer) { @@ -384,6 +415,8 @@ private void waitForServerDecom(HRegionServer excludeServer) { @Test public void testDecomServerExclusion() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer excludeServer = cluster.getRegionServer(0); List regions = excludeServer.getRegions(); @@ -422,10 +455,14 @@ public void testDecomServerExclusion() throws Exception { TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), Collections.emptyList()); + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testExcludeAndDecomServers() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); FileWriter fos = new FileWriter(excludeFile); @@ -456,6 +493,8 @@ public void testExcludeAndDecomServers() throws Exception { TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(), Collections.emptyList()); + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java index f73876556f4a..4351b75b9f56 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java @@ -34,13 +34,15 @@ import org.apache.hadoop.hbase.testclassification.MiscTests; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -57,12 +59,13 @@ public class TestRegionMover2 { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestRegionMover2.class); + @Rule + public TestName name = new TestName(); + private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final TableName TABLE_NAME = TableName.valueOf("testRegionMover2"); - @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(3); @@ -74,82 +77,82 @@ public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - @Before - public void setUp() throws Exception { - // Create a pre-split table just to populate some regions - Admin admin = TEST_UTIL.getAdmin(); - if (admin.tableExists(TABLE_NAME)) { - TEST_UTIL.deleteTable(TABLE_NAME); - } - TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(TABLE_NAME) + private void initTableRegions(TableName tableName) throws IOException { + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); int startKey = 0; int endKey = 80000; - admin.createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); + TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); } @Test public void testWithMergedRegions() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); Admin admin = TEST_UTIL.getAdmin(); - Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); + Table table = TEST_UTIL.getConnection().getTable(tableName); List puts = new ArrayList<>(); for (int i = 0; i < 10000; i++) { puts.add(new Put(Bytes.toBytes("rowkey_" + i)) .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); } table.put(puts); - admin.flush(TABLE_NAME); + admin.flush(tableName); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getAddress().toString(); int numRegions = regionServer.getNumberOfOnlineRegions(); List hRegions = regionServer.getRegions().stream() - .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(TABLE_NAME)) + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) .collect(Collectors.toList()); RegionMover.RegionMoverBuilder rmBuilder = new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) .maxthreads(8); try (RegionMover rm = rmBuilder.build()) { - LOG.debug("Unloading " + regionServer.getServerName()); + LOG.debug("Unloading {}", regionServer.getServerName()); rm.unload(); Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); - LOG.debug("Successfully Unloaded\nNow Loading"); + LOG.debug("Successfully Unloaded, now Loading"); admin.mergeRegionsAsync(new byte[][] { hRegions.get(0).getRegionInfo().getRegionName(), hRegions.get(1).getRegionInfo().getRegionName() }, true) .get(5, TimeUnit.SECONDS); Assert.assertTrue(rm.load()); Assert.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions()); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testWithSplitRegions() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); Admin admin = TEST_UTIL.getAdmin(); - Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); + Table table = TEST_UTIL.getConnection().getTable(tableName); List puts = new ArrayList<>(); for (int i = 10; i < 50000; i++) { puts.add(new Put(Bytes.toBytes(i)) .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); } table.put(puts); - admin.flush(TABLE_NAME); - admin.compact(TABLE_NAME); + admin.flush(tableName); + admin.compact(tableName); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getAddress().toString(); int numRegions = regionServer.getNumberOfOnlineRegions(); List hRegions = regionServer.getRegions().stream() - .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(TABLE_NAME)) + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) .collect(Collectors.toList()); RegionMover.RegionMoverBuilder rmBuilder = new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) .maxthreads(8); try (RegionMover rm = rmBuilder.build()) { - LOG.debug("Unloading " + regionServer.getServerName()); + LOG.debug("Unloading {}", regionServer.getServerName()); rm.unload(); Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); - LOG.debug("Successfully Unloaded\nNow Loading"); + LOG.debug("Successfully Unloaded, now Loading"); HRegion hRegion = hRegions.get(1); int startKey = 0; int endKey = Integer.MAX_VALUE; @@ -165,38 +168,44 @@ public void testWithSplitRegions() throws Exception { Assert.assertTrue(rm.load()); Assert.assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions()); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testFailedRegionMove() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); Admin admin = TEST_UTIL.getAdmin(); - Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); + Table table = TEST_UTIL.getConnection().getTable(tableName); List puts = new ArrayList<>(); for (int i = 0; i < 1000; i++) { puts.add(new Put(Bytes.toBytes("rowkey_" + i)) .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); } table.put(puts); - admin.flush(TABLE_NAME); + admin.flush(tableName); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getAddress().toString(); int numRegions = regionServer.getNumberOfOnlineRegions(); List hRegions = regionServer.getRegions().stream() - .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(TABLE_NAME)) + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) .collect(Collectors.toList()); RegionMover.RegionMoverBuilder rmBuilder = new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) .maxthreads(8); try (RegionMover rm = rmBuilder.build()) { - LOG.debug("Unloading " + regionServer.getServerName()); + LOG.debug("Unloading {}", regionServer.getServerName()); rm.unload(); Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); - LOG.debug("Successfully Unloaded\nNow Loading"); + LOG.debug("Successfully Unloaded, now Loading"); admin.offline(hRegions.get(0).getRegionInfo().getRegionName()); // loading regions will fail because of offline region Assert.assertFalse(rm.load()); } + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); } } From feced892b25e63b46c7525e5ff5d214294ba74a7 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Fri, 31 Jul 2020 21:35:11 +0530 Subject: [PATCH 5/5] moved table creation in Before and deletion in After --- .../hadoop/hbase/util/TestRegionMover1.java | 57 ++++--------------- .../hadoop/hbase/util/TestRegionMover2.java | 23 ++++---- 2 files changed, 24 insertions(+), 56 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover1.java index a5b79ee92eb4..8138f6b964b7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover1.java @@ -43,8 +43,10 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder; +import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; @@ -83,7 +85,9 @@ public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - private void initTableRegions(TableName tableName) throws IOException { + @Before + public void setUp() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); String startKey = "a"; @@ -91,10 +95,15 @@ private void initTableRegions(TableName tableName) throws IOException { TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); } + @After + public void tearDown() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); + } + @Test public void testWithAck() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getAddress().toString(); @@ -111,8 +120,6 @@ public void testWithAck() throws Exception { // Repeat the same load. It should be very fast because all regions are already moved. rm.load(); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } /** @@ -120,8 +127,6 @@ public void testWithAck() throws Exception { */ @Test public void testWithoutAck() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getAddress().toString(); @@ -148,8 +153,6 @@ public boolean evaluate() throws Exception { } }); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } /** @@ -158,8 +161,6 @@ public boolean evaluate() throws Exception { */ @Test public void testExclude() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); FileWriter fos = new FileWriter(excludeFile); @@ -184,14 +185,10 @@ public void testExclude() throws Exception { LOG.info("Before:" + regionsExcludeServer + " After:" + cluster.getRegionServer(1).getNumberOfOnlineRegions()); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testDesignatedFile() throws Exception{ - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "designated_file"); @@ -219,14 +216,10 @@ public void testDesignatedFile() throws Exception{ LOG.debug("Before:{} After:{}", regionsInDesignatedServer, designatedServer.getNumberOfOnlineRegions()); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testExcludeAndDesignated() throws Exception{ - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); // create designated file File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), @@ -271,14 +264,10 @@ public void testExcludeAndDesignated() throws Exception{ LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions()); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testRegionServerPort() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getHostname(); @@ -292,8 +281,6 @@ public void testRegionServerPort() throws Exception { if (originalPort != null) { conf.set(HConstants.REGIONSERVER_PORT, originalPort); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } /** @@ -301,8 +288,6 @@ public void testRegionServerPort() throws Exception { */ @Test public void testLoadMetaRegion() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() .map(t -> t.getRegionServer()) .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get(); @@ -317,8 +302,6 @@ public void testLoadMetaRegion() throws Exception { rm.load(); assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions()); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } /** @@ -326,8 +309,6 @@ public void testLoadMetaRegion() throws Exception { */ @Test public void testTargetServerDeadWhenLoading() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); String rsName = rs.getServerName().getAddress().toString(); Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); @@ -349,14 +330,10 @@ public void testTargetServerDeadWhenLoading() throws Exception { LOG.info("Loading to an inexist region server {}", inexistRsName); assertFalse(rm.load()); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testDecomServerExclusionWithAck() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer excludeServer = cluster.getRegionServer(1); List regions = excludeServer.getRegions(); @@ -396,8 +373,6 @@ public void testDecomServerExclusionWithAck() throws Exception { TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), Collections.emptyList()); - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } private void waitForServerDecom(HRegionServer excludeServer) { @@ -415,8 +390,6 @@ private void waitForServerDecom(HRegionServer excludeServer) { @Test public void testDecomServerExclusion() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HRegionServer excludeServer = cluster.getRegionServer(0); List regions = excludeServer.getRegions(); @@ -455,14 +428,10 @@ public void testDecomServerExclusion() throws Exception { TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), Collections.emptyList()); - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testExcludeAndDecomServers() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); FileWriter fos = new FileWriter(excludeFile); @@ -493,8 +462,6 @@ public void testExcludeAndDecomServers() throws Exception { TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(), Collections.emptyList()); - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java index 4351b75b9f56..743fa99611a5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java @@ -32,8 +32,10 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; @@ -42,7 +44,6 @@ import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -77,7 +78,9 @@ public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - private void initTableRegions(TableName tableName) throws IOException { + @Before + public void setUp() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); int startKey = 0; @@ -85,10 +88,16 @@ private void initTableRegions(TableName tableName) throws IOException { TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); } + @After + public void tearDown() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); + } + @Test public void testWithMergedRegions() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); Admin admin = TEST_UTIL.getAdmin(); Table table = TEST_UTIL.getConnection().getTable(tableName); @@ -119,14 +128,11 @@ public void testWithMergedRegions() throws Exception { Assert.assertTrue(rm.load()); Assert.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions()); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testWithSplitRegions() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); Admin admin = TEST_UTIL.getAdmin(); Table table = TEST_UTIL.getConnection().getTable(tableName); @@ -168,14 +174,11 @@ public void testWithSplitRegions() throws Exception { Assert.assertTrue(rm.load()); Assert.assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions()); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } @Test public void testFailedRegionMove() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); - initTableRegions(tableName); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); Admin admin = TEST_UTIL.getAdmin(); Table table = TEST_UTIL.getConnection().getTable(tableName); @@ -204,8 +207,6 @@ public void testFailedRegionMove() throws Exception { // loading regions will fail because of offline region Assert.assertFalse(rm.load()); } - TEST_UTIL.getAdmin().disableTable(tableName); - TEST_UTIL.getAdmin().deleteTable(tableName); } }