From 327d0545acdd7d2954465af19378c4d80102a6ff Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Mon, 22 Apr 2024 10:20:05 -0700 Subject: [PATCH 1/5] HDDS-10559. Add a warning or a check to run repair tool as System user --- .../hadoop/ozone/repair/OzoneRepair.java | 39 ++++++++ .../hadoop/ozone/repair/TestOzoneRepair.java | 94 +++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java index 3bbbded58028..e18cfd953743 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java @@ -22,8 +22,11 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.tracing.TracingUtil; import picocli.CommandLine; +import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; + /** * Ozone Repair Command line tool. */ @@ -33,6 +36,13 @@ mixinStandardHelpOptions = true) public class OzoneRepair extends GenericCli { + public static final String WARNING_SYS_USER_MESSAGE = + "ATTENTION: You are currently logged in as user '%s'. Ozone typically runs as user '%s'." + + " If you proceed with this command, it may change the ownership of RocksDB files " + + " used by the Ozone Manager (OM). This ownership change could prevent OM from starting successfully." + + " Are you sure you want to continue (y/N)? "; + + private OzoneConfiguration ozoneConf; public OzoneRepair() { @@ -61,4 +71,33 @@ public OzoneConfiguration getOzoneConf() { public static void main(String[] argv) throws Exception { new OzoneRepair().run(argv); } + + @Override + public int execute(String[] argv) { + String currentUser = getSystemUserName(); + boolean shouldProceed = true; + if (!currentUser.equals(OZONE_DEFAULT_USER)) { + String s = getConsoleReadLineWithFormat(currentUser, OZONE_DEFAULT_USER); + shouldProceed = Boolean.parseBoolean(s) || "y".equalsIgnoreCase(s); + } + if (!shouldProceed) { + System.out.println("Aborting command."); + return 1; + } + System.out.println("Run as user: " + currentUser); + + TracingUtil.initTracing("shell", createOzoneConfiguration()); + String spanName = "ozone repair " + String.join(" ", argv); + return TracingUtil.executeInNewSpan(spanName, + () -> super.execute(argv)); + } + + public String getSystemUserName() { + return System.getProperty("user.name"); + } + + public String getConsoleReadLineWithFormat(String currentUser, String defaultUser) { + return System.console().readLine(String.format(WARNING_SYS_USER_MESSAGE, currentUser, defaultUser)); + } + } diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java new file mode 100644 index 000000000000..16d7167b2dc3 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.repair; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +/** + * Tests the ozone repair command. + */ +public class TestOzoneRepair { + + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private final ByteArrayOutputStream err = new ByteArrayOutputStream(); + private static final PrintStream OLD_OUT = System.out; + private static final PrintStream OLD_ERR = System.err; + private static final String DEFAULT_ENCODING = UTF_8.name(); + + @BeforeEach + public void setup() throws Exception { + System.setOut(new PrintStream(out, false, DEFAULT_ENCODING)); + System.setErr(new PrintStream(err, false, DEFAULT_ENCODING)); + } + + @AfterEach + public void reset() { + // reset stream after each unit test + out.reset(); + err.reset(); + + // restore system streams + System.setOut(OLD_OUT); + System.setErr(OLD_ERR); + } + + @Test + void testOzoneRepairWhenUserIsOzoneDefaultUser() throws Exception { + OzoneRepair ozoneRepair = spy(new OzoneRepair()); + doReturn(OZONE_DEFAULT_USER).when(ozoneRepair).getSystemUserName(); + + ozoneRepair.execute(new String[]{}); + assertThat(out.toString(DEFAULT_ENCODING)).contains("Run as user: " + OZONE_DEFAULT_USER); + } + + @Test + void testOzoneRepairWhenUserIsNotOzoneDefaultUserAndDeclinesToProceed() throws Exception { + OzoneRepair ozoneRepair = spy(new OzoneRepair()); + doReturn("non-ozone").when(ozoneRepair).getSystemUserName(); + doReturn("N").when(ozoneRepair).getConsoleReadLineWithFormat(anyString(), anyString()); + + int res = ozoneRepair.execute(new String[]{}); + assertEquals(1, res); + assertThat(out.toString(DEFAULT_ENCODING)).contains("Aborting command."); + } + + @Test + void testOzoneRepairWhenUserIsNotOzoneDefaultUserAndAgreesToProceed() throws Exception { + OzoneRepair ozoneRepair = spy(new OzoneRepair()); + String currentUser = "non-ozone"; + doReturn(currentUser).when(ozoneRepair).getSystemUserName(); + doReturn("y").when(ozoneRepair).getConsoleReadLineWithFormat(anyString(), anyString()); + + ozoneRepair.execute(new String[]{}); + assertThat(out.toString(DEFAULT_ENCODING)).contains("Run as user: " + currentUser); + } + +} From fae7cef3b254485b0184f819d11f6c3e174af17e Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Mon, 29 Apr 2024 17:39:13 -0700 Subject: [PATCH 2/5] Update command message, removed unnecessary tracing --- .../hadoop/ozone/repair/OzoneRepair.java | 23 ++++----------- .../hadoop/ozone/repair/TestOzoneRepair.java | 29 +++++++------------ 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java index e18cfd953743..68bee9029f4b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java @@ -22,11 +22,8 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.tracing.TracingUtil; import picocli.CommandLine; -import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; - /** * Ozone Repair Command line tool. */ @@ -37,9 +34,7 @@ public class OzoneRepair extends GenericCli { public static final String WARNING_SYS_USER_MESSAGE = - "ATTENTION: You are currently logged in as user '%s'. Ozone typically runs as user '%s'." + - " If you proceed with this command, it may change the ownership of RocksDB files " + - " used by the Ozone Manager (OM). This ownership change could prevent OM from starting successfully." + + "ATTENTION: Running as user '%s'. Make sure this is the same user used to run the Ozone process." + " Are you sure you want to continue (y/N)? "; @@ -75,29 +70,23 @@ public static void main(String[] argv) throws Exception { @Override public int execute(String[] argv) { String currentUser = getSystemUserName(); - boolean shouldProceed = true; - if (!currentUser.equals(OZONE_DEFAULT_USER)) { - String s = getConsoleReadLineWithFormat(currentUser, OZONE_DEFAULT_USER); - shouldProceed = Boolean.parseBoolean(s) || "y".equalsIgnoreCase(s); - } + String s = getConsoleReadLineWithFormat(currentUser); + boolean shouldProceed = Boolean.parseBoolean(s) || "y".equalsIgnoreCase(s); if (!shouldProceed) { System.out.println("Aborting command."); return 1; } System.out.println("Run as user: " + currentUser); - TracingUtil.initTracing("shell", createOzoneConfiguration()); - String spanName = "ozone repair " + String.join(" ", argv); - return TracingUtil.executeInNewSpan(spanName, - () -> super.execute(argv)); + return super.execute(argv); } public String getSystemUserName() { return System.getProperty("user.name"); } - public String getConsoleReadLineWithFormat(String currentUser, String defaultUser) { - return System.console().readLine(String.format(WARNING_SYS_USER_MESSAGE, currentUser, defaultUser)); + public String getConsoleReadLineWithFormat(String currentUser) { + return System.console().readLine(String.format(WARNING_SYS_USER_MESSAGE, currentUser)); } } diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java index 16d7167b2dc3..0031d4647726 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.repair; +import org.jooq.meta.derby.sys.Sys; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -25,7 +26,6 @@ import java.io.PrintStream; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.anyString; @@ -43,10 +43,14 @@ public class TestOzoneRepair { private static final PrintStream OLD_ERR = System.err; private static final String DEFAULT_ENCODING = UTF_8.name(); + private static final String OZONE_USER = "ozone"; + private static final String OLD_USER = System.getProperty("user.name"); + @BeforeEach public void setup() throws Exception { System.setOut(new PrintStream(out, false, DEFAULT_ENCODING)); System.setErr(new PrintStream(err, false, DEFAULT_ENCODING)); + System.setProperty("user.name", OZONE_USER); } @AfterEach @@ -58,22 +62,13 @@ public void reset() { // restore system streams System.setOut(OLD_OUT); System.setErr(OLD_ERR); + System.setProperty("user.name", OLD_USER); } @Test - void testOzoneRepairWhenUserIsOzoneDefaultUser() throws Exception { - OzoneRepair ozoneRepair = spy(new OzoneRepair()); - doReturn(OZONE_DEFAULT_USER).when(ozoneRepair).getSystemUserName(); - - ozoneRepair.execute(new String[]{}); - assertThat(out.toString(DEFAULT_ENCODING)).contains("Run as user: " + OZONE_DEFAULT_USER); - } - - @Test - void testOzoneRepairWhenUserIsNotOzoneDefaultUserAndDeclinesToProceed() throws Exception { + void testOzoneRepairWhenUserIsRemindedSystemUserAndDeclinesToProceed() throws Exception { OzoneRepair ozoneRepair = spy(new OzoneRepair()); - doReturn("non-ozone").when(ozoneRepair).getSystemUserName(); - doReturn("N").when(ozoneRepair).getConsoleReadLineWithFormat(anyString(), anyString()); + doReturn("N").when(ozoneRepair).getConsoleReadLineWithFormat(anyString()); int res = ozoneRepair.execute(new String[]{}); assertEquals(1, res); @@ -81,14 +76,12 @@ void testOzoneRepairWhenUserIsNotOzoneDefaultUserAndDeclinesToProceed() throws E } @Test - void testOzoneRepairWhenUserIsNotOzoneDefaultUserAndAgreesToProceed() throws Exception { + void testOzoneRepairWhenUserIsRemindedSystemUserAndAgreesToProceed() throws Exception { OzoneRepair ozoneRepair = spy(new OzoneRepair()); - String currentUser = "non-ozone"; - doReturn(currentUser).when(ozoneRepair).getSystemUserName(); - doReturn("y").when(ozoneRepair).getConsoleReadLineWithFormat(anyString(), anyString()); + doReturn("y").when(ozoneRepair).getConsoleReadLineWithFormat(anyString()); ozoneRepair.execute(new String[]{}); - assertThat(out.toString(DEFAULT_ENCODING)).contains("Run as user: " + currentUser); + assertThat(out.toString(DEFAULT_ENCODING)).contains("Run as user: " + OZONE_USER); } } From 67fb31f8ddf93ee46bbbf6db11c6278e319fe647 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Mon, 29 Apr 2024 20:12:45 -0700 Subject: [PATCH 3/5] fix checkstyle --- .../java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java index 0031d4647726..1f19c9330211 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.ozone.repair; -import org.jooq.meta.derby.sys.Sys; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; From 8781efac93a221ce28a621d776f047ed684e7f76 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Fri, 10 May 2024 10:39:15 -0700 Subject: [PATCH 4/5] Use Scanner to scan terminal input, make sure the prompt goes to stderr --- .../org/apache/hadoop/ozone/repair/OzoneRepair.java | 12 +++++++----- .../apache/hadoop/ozone/repair/TestOzoneRepair.java | 13 +++++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java index 68bee9029f4b..3b745d2c1d95 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java @@ -24,6 +24,9 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import picocli.CommandLine; +import java.nio.charset.StandardCharsets; +import java.util.Scanner; + /** * Ozone Repair Command line tool. */ @@ -34,7 +37,7 @@ public class OzoneRepair extends GenericCli { public static final String WARNING_SYS_USER_MESSAGE = - "ATTENTION: Running as user '%s'. Make sure this is the same user used to run the Ozone process." + + "ATTENTION: Running as user %s. Make sure this is the same user used to run the Ozone process." + " Are you sure you want to continue (y/N)? "; @@ -70,9 +73,7 @@ public static void main(String[] argv) throws Exception { @Override public int execute(String[] argv) { String currentUser = getSystemUserName(); - String s = getConsoleReadLineWithFormat(currentUser); - boolean shouldProceed = Boolean.parseBoolean(s) || "y".equalsIgnoreCase(s); - if (!shouldProceed) { + if (!( "y".equalsIgnoreCase(getConsoleReadLineWithFormat(currentUser)))) { System.out.println("Aborting command."); return 1; } @@ -86,7 +87,8 @@ public String getSystemUserName() { } public String getConsoleReadLineWithFormat(String currentUser) { - return System.console().readLine(String.format(WARNING_SYS_USER_MESSAGE, currentUser)); + System.err.printf(WARNING_SYS_USER_MESSAGE, currentUser); + return (new Scanner(System.in, StandardCharsets.UTF_8.name())).nextLine().trim(); } } diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java index 1f19c9330211..cfcbf4ac352b 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java @@ -21,6 +21,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.PrintStream; @@ -66,21 +67,25 @@ public void reset() { @Test void testOzoneRepairWhenUserIsRemindedSystemUserAndDeclinesToProceed() throws Exception { - OzoneRepair ozoneRepair = spy(new OzoneRepair()); - doReturn("N").when(ozoneRepair).getConsoleReadLineWithFormat(anyString()); + OzoneRepair ozoneRepair = new OzoneRepair(); + System.setIn(new ByteArrayInputStream("N".getBytes(DEFAULT_ENCODING))); int res = ozoneRepair.execute(new String[]{}); assertEquals(1, res); assertThat(out.toString(DEFAULT_ENCODING)).contains("Aborting command."); + // prompt should contain the current user name as well + assertThat(err.toString(DEFAULT_ENCODING)).contains("ATTENTION: Running as user " + OZONE_USER); } @Test void testOzoneRepairWhenUserIsRemindedSystemUserAndAgreesToProceed() throws Exception { - OzoneRepair ozoneRepair = spy(new OzoneRepair()); - doReturn("y").when(ozoneRepair).getConsoleReadLineWithFormat(anyString()); + OzoneRepair ozoneRepair = new OzoneRepair(); + System.setIn(new ByteArrayInputStream("y".getBytes(DEFAULT_ENCODING))); ozoneRepair.execute(new String[]{}); assertThat(out.toString(DEFAULT_ENCODING)).contains("Run as user: " + OZONE_USER); + // prompt should contain the current user name as well + assertThat(err.toString(DEFAULT_ENCODING)).contains("ATTENTION: Running as user " + OZONE_USER); } } From 52cf3db8f5705e4dcbf764b10fc61fe93160d623 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Fri, 10 May 2024 10:42:05 -0700 Subject: [PATCH 5/5] remove unused import --- .../main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java | 2 +- .../java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java index 3b745d2c1d95..166445228089 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java @@ -73,7 +73,7 @@ public static void main(String[] argv) throws Exception { @Override public int execute(String[] argv) { String currentUser = getSystemUserName(); - if (!( "y".equalsIgnoreCase(getConsoleReadLineWithFormat(currentUser)))) { + if (!("y".equalsIgnoreCase(getConsoleReadLineWithFormat(currentUser)))) { System.out.println("Aborting command."); return 1; } diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java index cfcbf4ac352b..272bf24c066e 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestOzoneRepair.java @@ -28,9 +28,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; /** * Tests the ozone repair command.