From 3f90162b40d640b04bc14c4b054eb64ba2919bbe Mon Sep 17 00:00:00 2001 From: LiangJun He <2009bjhlj@gmail.com> Date: Mon, 14 Feb 2022 18:12:59 +0800 Subject: [PATCH 1/3] HBASE-26673 Implement a shell command for change SFT implementation --- hbase-shell/src/main/ruby/hbase/admin.rb | 12 ++++ hbase-shell/src/main/ruby/shell.rb | 8 +++ .../main/ruby/shell/commands/change_sft.rb | 54 ++++++++++++++++++ .../hbase/client/TestChangeSftShell.java | 46 +++++++++++++++ .../test/ruby/shell/sftchange_shell_test.rb | 56 +++++++++++++++++++ 5 files changed, 176 insertions(+) create mode 100644 hbase-shell/src/main/ruby/shell/commands/change_sft.rb create mode 100644 hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java create mode 100644 hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 62fc7fed7d52..e60a5ce77736 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1825,6 +1825,18 @@ def to_server_names(server_names) java.util.Arrays.asList(server_names) end end + + #---------------------------------------------------------------------------------------------- + # Change table's sft + def modify_table_sft(tableName, sft) + @admin.modifyTableStoreFileTracker(tableName, sft) + end + + #---------------------------------------------------------------------------------------------- + # Change table column family's sft + def modify_table_family_sft(tableName, family_bytes, sft) + @admin.modifyColumnFamilyStoreFileTracker(tableName, family_bytes, sft) + end end # rubocop:enable Metrics/ClassLength end diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index ba7481eb0bef..352829c531c2 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -625,3 +625,11 @@ def self.exception_handler(hide_traceback) get_namespace_rsgroup ] ) + +Shell.load_command_group( + 'storefiletracker', + full_name: 'StoreFileTracker', + commands: %w[ + change_sft + ] +) diff --git a/hbase-shell/src/main/ruby/shell/commands/change_sft.rb b/hbase-shell/src/main/ruby/shell/commands/change_sft.rb new file mode 100644 index 000000000000..5579687f1d76 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/change_sft.rb @@ -0,0 +1,54 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ChangeSft < Command + def help + <<-EOF +Change table's or table column family's sft. Examples: + + hbase> change_sft 't1','FILE' + hbase> change_sft 'ns:.*','FILE' + hbase> change_sft 't2','cf1','FILE' +EOF + end + + def command(*args) + arg_length = args.length + if arg_length == 2 + tableList = admin.list(args[0]) + sft = args[1] + tableList.each do |table| + tableName = TableName.valueOf(table) + admin.modify_table_sft(tableName, sft) + end + elsif arg_length == 3 + tableName = TableName.valueOf(args[0]) + family = args[1] + family_bytes = family.to_java_bytes + sft = args[2] + admin.modify_table_family_sft(tableName, family_bytes, sft) + else + raise(ArgumentError, 'Argument length should be two or three.') + end + end + end + end +end diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java new file mode 100644 index 000000000000..5e96e27bf1b1 --- /dev/null +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.experimental.categories.Category; + +@Category({ ClientTests.class, LargeTests.class }) +public class TestChangeSftShell extends AbstractTestShell { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestChangeSftShell.class); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + setUpConfig(); + + TEST_UTIL.startMiniCluster(3); + + setUpJRubyRuntime(); + } + + @Override + protected String getIncludeList() { + return "sftchange_shell_test.rb"; + } +} diff --git a/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb b/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb new file mode 100644 index 000000000000..cc7f9d13ec0f --- /dev/null +++ b/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb @@ -0,0 +1,56 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'hbase_constants' +require 'hbase_shell' + +class SftChangeTest < Test::Unit::TestCase + def setup + @hbase = ::Hbase::Hbase.new($TEST_CLUSTER.getConfiguration) + @shell = Shell::Shell.new(@hbase) + connection = $TEST_CLUSTER.getConnection + @admin = connection.getAdmin + end + + define_test "Change table's sft" do + table = 'test_table1' + family = 'f1' + change_sft = 'FILE' + @shell.command('create', table, family) + @shell.command('change_sft', table, change_sft) + table_sft = @admin.getDescriptor(TableName.valueOf(table)).getValue('hbase.store.file-tracker.impl') + assert_equal(change_sft, table_sft) + @shell.command(:disable, table) + @shell.command(:drop, table) + end + + define_test "Change table family's sft" do + table = 'test_table2' + family = 'f1' + change_sft = 'FILE' + @shell.command('create', table, family) + @shell.command('change_sft', table, family, change_sft) + family_bytes = family.to_java_bytes + cfd = @admin.getDescriptor(TableName.valueOf(table)).getColumnFamily(family_bytes) + table_family_sft = cfd.getConfigurationValue('hbase.store.file-tracker.impl') + assert_equal(change_sft, table_family_sft) + @shell.command(:disable, table) + @shell.command(:drop, table) + end +end From c430ffd944f11e2bbdd3e5f08d68047d1f7ef82f Mon Sep 17 00:00:00 2001 From: LiangJun He <2009bjhlj@gmail.com> Date: Tue, 15 Feb 2022 08:34:28 +0800 Subject: [PATCH 2/3] add change ns all table's sft ut --- .../test/ruby/shell/sftchange_shell_test.rb | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb b/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb index cc7f9d13ec0f..becc943fa0b0 100644 --- a/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb +++ b/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb @@ -40,7 +40,25 @@ def setup @shell.command(:drop, table) end - define_test "Change table family's sft" do + define_test "Change ns all table's sft" do + table1 = 'default:test_table1' + table2 = 'default:test_table2' + family = 'f1' + change_sft = 'FILE' + @shell.command('create', table1, family) + @shell.command('create', table2, family) + @shell.command('change_sft', 'default:.*', change_sft) + table1_sft = @admin.getDescriptor(TableName.valueOf(table1)).getValue('hbase.store.file-tracker.impl') + assert_equal(change_sft, table1_sft) + table2_sft = @admin.getDescriptor(TableName.valueOf(table2)).getValue('hbase.store.file-tracker.impl') + assert_equal(change_sft, table2_sft) + @shell.command(:disable, table1) + @shell.command(:drop, table1) + @shell.command(:disable, table2) + @shell.command(:drop, table2) + end + + define_test "Change table column family's sft" do table = 'test_table2' family = 'f1' change_sft = 'FILE' From 1e5538875b2c0f04fb7e5120bbdc85301a6a8be2 Mon Sep 17 00:00:00 2001 From: LiangJun He <2009bjhlj@gmail.com> Date: Tue, 15 Feb 2022 11:17:50 +0800 Subject: [PATCH 3/3] add shell 'change_sft_all' support match table regex --- hbase-shell/src/main/ruby/shell.rb | 1 + .../main/ruby/shell/commands/change_sft.rb | 8 +-- .../ruby/shell/commands/change_sft_all.rb | 58 +++++++++++++++++++ .../test/ruby/shell/sftchange_shell_test.rb | 18 ------ 4 files changed, 61 insertions(+), 24 deletions(-) create mode 100644 hbase-shell/src/main/ruby/shell/commands/change_sft_all.rb diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 352829c531c2..2c553d229b06 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -631,5 +631,6 @@ def self.exception_handler(hide_traceback) full_name: 'StoreFileTracker', commands: %w[ change_sft + change_sft_all ] ) diff --git a/hbase-shell/src/main/ruby/shell/commands/change_sft.rb b/hbase-shell/src/main/ruby/shell/commands/change_sft.rb index 5579687f1d76..eb96f426a6a1 100644 --- a/hbase-shell/src/main/ruby/shell/commands/change_sft.rb +++ b/hbase-shell/src/main/ruby/shell/commands/change_sft.rb @@ -25,7 +25,6 @@ def help Change table's or table column family's sft. Examples: hbase> change_sft 't1','FILE' - hbase> change_sft 'ns:.*','FILE' hbase> change_sft 't2','cf1','FILE' EOF end @@ -33,12 +32,9 @@ def help def command(*args) arg_length = args.length if arg_length == 2 - tableList = admin.list(args[0]) + tableName = TableName.valueOf(args[0]) sft = args[1] - tableList.each do |table| - tableName = TableName.valueOf(table) - admin.modify_table_sft(tableName, sft) - end + admin.modify_table_sft(tableName, sft) elsif arg_length == 3 tableName = TableName.valueOf(args[0]) family = args[1] diff --git a/hbase-shell/src/main/ruby/shell/commands/change_sft_all.rb b/hbase-shell/src/main/ruby/shell/commands/change_sft_all.rb new file mode 100644 index 000000000000..6e348195babe --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/change_sft_all.rb @@ -0,0 +1,58 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ChangeSftAll < Command + def help + <<-EOF +Change all of the tables's sft matching the given regex: + + hbase> change_sft_all 't.*','FILE' + hbase> change_sft_all 'ns:.*','FILE' + hbase> change_sft_all 'ns:t.*','FILE' +EOF + end + + def command(*args) + arg_length = args.length + if arg_length == 2 + tableRegex = args[0] + tableList = admin.list(tableRegex) + count = tableList.size + sft = args[1] + tableList.each do |table| + formatter.row([table]) + end + puts "\nChange the above #{count} tables's sft (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{tableRegex}" if count == 0 + return unless answer =~ /y.*/i + tableList.each do |table| + tableName = TableName.valueOf(table) + admin.modify_table_sft(tableName, sft) + end + else + raise(ArgumentError, 'Argument length should be two.') + end + end + end + end +end diff --git a/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb b/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb index becc943fa0b0..220b624e2b47 100644 --- a/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb +++ b/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb @@ -40,24 +40,6 @@ def setup @shell.command(:drop, table) end - define_test "Change ns all table's sft" do - table1 = 'default:test_table1' - table2 = 'default:test_table2' - family = 'f1' - change_sft = 'FILE' - @shell.command('create', table1, family) - @shell.command('create', table2, family) - @shell.command('change_sft', 'default:.*', change_sft) - table1_sft = @admin.getDescriptor(TableName.valueOf(table1)).getValue('hbase.store.file-tracker.impl') - assert_equal(change_sft, table1_sft) - table2_sft = @admin.getDescriptor(TableName.valueOf(table2)).getValue('hbase.store.file-tracker.impl') - assert_equal(change_sft, table2_sft) - @shell.command(:disable, table1) - @shell.command(:drop, table1) - @shell.command(:disable, table2) - @shell.command(:drop, table2) - end - define_test "Change table column family's sft" do table = 'test_table2' family = 'f1'