From e8f3e579eb420a3d3dc2ab65b806a30b808b31e0 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 26 Mar 2025 17:14:03 +0100 Subject: [PATCH 1/4] HDDS-12702. Move checknative under ozone debug --- hadoop-hdds/docs/content/feature/ErasureCoding.md | 2 +- hadoop-ozone/dist/src/shell/ozone/ozone | 9 ++------- .../hadoop/ozone/shell/checknative/CheckNative.java | 13 ++++++------- .../hadoop/ozone/checknative/TestCheckNative.java | 7 +++---- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/docs/content/feature/ErasureCoding.md b/hadoop-hdds/docs/content/feature/ErasureCoding.md index c174da2bc644..9e60c3a923a2 100644 --- a/hadoop-hdds/docs/content/feature/ErasureCoding.md +++ b/hadoop-hdds/docs/content/feature/ErasureCoding.md @@ -246,7 +246,7 @@ The default value of java.library.path depends on the OS and Java version. For e You can check if ISA-L is accessible to Ozone by running the following command: ```shell -ozone checknative +ozone debug checknative ``` diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 08bbf1acfe63..40485a952354 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -59,7 +59,6 @@ function ozone_usage ozone_add_subcommand "admin" client "Ozone admin tool" ozone_add_subcommand "debug" client "Ozone debug tool" ozone_add_subcommand "repair" client "Ozone repair tool" - ozone_add_subcommand "checknative" client "checks if native libraries are loaded" ozone_add_subcommand "ratis" client "Ozone ratis tool" ozone_generate_usage "${OZONE_SHELL_EXECNAME}" false @@ -224,10 +223,6 @@ function ozonecmd_case OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; - checknative) - OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.checknative.CheckNative - OZONE_RUN_ARTIFACT_NAME="ozone-tools" - ;; ratis) OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.OzoneRatis OZONE_RUN_ARTIFACT_NAME="ozone-tools" @@ -305,8 +300,8 @@ fi OZONE_SUBCMD=$1 -if [[ "$OZONE_SUBCMD" == "auditparser" ]]; then - echo "warning: 'ozone auditparser' is deprecated, use 'ozone debug auditparser' instead." +if [[ "$OZONE_SUBCMD" == "auditparser" ]] || [[ "$OZONE_SUBCMD" == "checknative" ]]; then + echo "warning: 'ozone $OZONE_SUBCMD' is deprecated, use 'ozone debug $OZONE_SUBCMD' instead." OZONE_SUBCMD="debug" else shift diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java index e3bdd475c998..7411ffc751e5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java @@ -21,22 +21,21 @@ import java.util.Collections; import java.util.concurrent.Callable; -import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.AbstractSubcommand; +import org.apache.hadoop.hdds.cli.DebugSubcommand; import org.apache.hadoop.hdds.utils.NativeLibraryLoader; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.kohsuke.MetaInfServices; import picocli.CommandLine; /** * CLI command to check if native libraries are loaded. */ -@CommandLine.Command(name = "ozone checknative", +@CommandLine.Command(name = "checknative", description = "Checks if native libraries are loaded") -public class CheckNative extends GenericCli implements Callable { - - public static void main(String[] argv) { - new CheckNative().run(argv); - } +@MetaInfServices(DebugSubcommand.class) +public class CheckNative extends AbstractSubcommand implements Callable, DebugSubcommand { @Override public Void call() throws Exception { diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java index ab066129ad5f..ef2bd2451daf 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java @@ -24,6 +24,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.UnsupportedEncodingException; +import org.apache.hadoop.ozone.debug.OzoneDebug; import org.apache.hadoop.ozone.shell.checknative.CheckNative; import org.apache.ozone.test.tag.Native; import org.junit.jupiter.api.AfterAll; @@ -52,8 +53,7 @@ public static void init() throws UnsupportedEncodingException { @Test public void testCheckNativeNotLoaded() throws UnsupportedEncodingException { outputStream.reset(); - new CheckNative() - .run(new String[] {}); + new OzoneDebug().getCmd().execute("checknative"); // trims multiple spaces String stdOut = outputStream.toString(DEFAULT_ENCODING) .replaceAll(" +", " "); @@ -67,8 +67,7 @@ public void testCheckNativeNotLoaded() throws UnsupportedEncodingException { @Test public void testCheckNativeRocksToolsLoaded() throws UnsupportedEncodingException { outputStream.reset(); - new CheckNative() - .run(new String[] {}); + new OzoneDebug().getCmd().execute("checknative"); // trims multiple spaces String stdOut = outputStream.toString(DEFAULT_ENCODING) .replaceAll(" +", " "); From 1372e04c61f183986fd769c62554ac683d900821 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 26 Mar 2025 17:23:09 +0100 Subject: [PATCH 2/4] improve TestCheckNative --- .../ozone/checknative/TestCheckNative.java | 67 ++++++++----------- 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java index ef2bd2451daf..3f382e9017f2 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java @@ -17,19 +17,16 @@ package org.apache.hadoop.ozone.checknative; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; import static org.assertj.core.api.Assertions.assertThat; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.debug.OzoneDebug; import org.apache.hadoop.ozone.shell.checknative.CheckNative; +import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Native; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** @@ -37,53 +34,43 @@ */ public class TestCheckNative { - private static PrintStream psBackup; - private static ByteArrayOutputStream outputStream; + private GenericTestUtils.PrintStreamCapturer out; - private static final String DEFAULT_ENCODING = UTF_8.name(); - - @BeforeAll - public static void init() throws UnsupportedEncodingException { - psBackup = System.out; - outputStream = new ByteArrayOutputStream(); - PrintStream psOut = new PrintStream(outputStream, false, DEFAULT_ENCODING); - System.setOut(psOut); + @BeforeEach + void init() { + out = GenericTestUtils.captureOut(); } @Test - public void testCheckNativeNotLoaded() throws UnsupportedEncodingException { - outputStream.reset(); - new OzoneDebug().getCmd().execute("checknative"); - // trims multiple spaces - String stdOut = outputStream.toString(DEFAULT_ENCODING) - .replaceAll(" +", " "); - assertThat(stdOut).contains("Native library checking:"); - assertThat(stdOut).contains("hadoop: false"); - assertThat(stdOut).contains("ISA-L: false"); - assertThat(stdOut).contains("rocks-tools: false"); + void testCheckNativeNotLoaded() { + executeCheckNative(); + assertOutput(false); } @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) @Test - public void testCheckNativeRocksToolsLoaded() throws UnsupportedEncodingException { - outputStream.reset(); - new OzoneDebug().getCmd().execute("checknative"); + void testCheckNativeRocksToolsLoaded() { + executeCheckNative(); + assertOutput(true); + } + + private void assertOutput(boolean expectedRocksNative) { // trims multiple spaces - String stdOut = outputStream.toString(DEFAULT_ENCODING) - .replaceAll(" +", " "); - assertThat(stdOut).contains("Native library checking:"); - assertThat(stdOut).contains("hadoop: false"); - assertThat(stdOut).contains("ISA-L: false"); - assertThat(stdOut).contains("rocks-tools: true"); + String stdOut = out.get() + .replaceAll(" +", " "); + assertThat(stdOut) + .contains("Native library checking:") + .contains("hadoop: false") + .contains("ISA-L: false") + .contains("rocks-tools: " + expectedRocksNative); } @AfterEach - public void setUp() { - outputStream.reset(); + void setUp() { + IOUtils.closeQuietly(out); } - @AfterAll - public static void tearDown() { - System.setOut(psBackup); + private static void executeCheckNative() { + new OzoneDebug().getCmd().execute("checknative"); } } From a885b84d4d05bd3f01618935c16c707795de6226 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 26 Mar 2025 17:34:41 +0100 Subject: [PATCH 3/4] move CheckNative to org.apache.hadoop.ozone.debug package --- .../checknative => debug}/CheckNative.java | 2 +- .../ozone/shell/checknative/package-info.java | 21 ------------------- .../TestCheckNative.java | 4 +--- 3 files changed, 2 insertions(+), 25 deletions(-) rename hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{shell/checknative => debug}/CheckNative.java (98%) delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/package-info.java rename hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/{checknative => debug}/TestCheckNative.java (93%) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/CheckNative.java similarity index 98% rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/CheckNative.java index 7411ffc751e5..1225122b4e4d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/CheckNative.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.shell.checknative; +package org.apache.hadoop.ozone.debug; import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/package-info.java deleted file mode 100644 index a4ca463b8347..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package for commands related to checking native libraries. - */ -package org.apache.hadoop.ozone.shell.checknative; diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java similarity index 93% rename from hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java index 3f382e9017f2..846557a510a2 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java @@ -15,14 +15,12 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.checknative; +package org.apache.hadoop.ozone.debug; import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; import static org.assertj.core.api.Assertions.assertThat; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.debug.OzoneDebug; -import org.apache.hadoop.ozone.shell.checknative.CheckNative; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Native; import org.junit.jupiter.api.AfterEach; From e0823cd3066fef0a75bc4e45015d40b84ad3b755 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 26 Mar 2025 17:35:16 +0100 Subject: [PATCH 4/4] reduce test class visibility --- .../java/org/apache/hadoop/ozone/debug/TestCheckNative.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java index 846557a510a2..eb6009a686bc 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java @@ -30,7 +30,7 @@ /** * Tests for {@link CheckNative}. */ -public class TestCheckNative { +class TestCheckNative { private GenericTestUtils.PrintStreamCapturer out;