From 96f4deee647dd7f7a76ca321efcb8e7d3f386e2c Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Fri, 6 Sep 2024 11:26:51 +0530 Subject: [PATCH 01/10] HDDS-10479. Fix RaftPeerId generated by command of raftMetaConf to use real PeerId --- .../hdds/scm/cli/ratis/RatisCommands.java | 54 +++++ .../RatisLocalRaftMetaConfSubCommand.java | 153 ++++++++++++++ .../cli/ratis/local/RatisLocalSubCommand.java | 57 ++++++ .../TestRatisLocalRaftMetaConfSubCommand.java | 188 ++++++++++++++++++ 4 files changed, 452 insertions(+) create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/RatisCommands.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalSubCommand.java create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/RatisCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/RatisCommands.java new file mode 100644 index 000000000000..b9918af65ba5 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/RatisCommands.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.ratis; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.OzoneAdmin; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.kohsuke.MetaInfServices; +import picocli.CommandLine; + +import java.util.concurrent.Callable; + +/** + * Subcommand for ratis related operations. + */ +@CommandLine.Command( + name = "ratis", + description = "Ratis specific operations", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class +) +@MetaInfServices(SubcommandWithParent.class) +public class RatisCommands implements Callable, SubcommandWithParent { + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public Void call() throws Exception { + GenericCli.missingSubcommand(spec); + return null; + } + + @Override + public Class getParentType() { + return OzoneAdmin.class; + } +} diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java new file mode 100644 index 000000000000..f2a4c0890331 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.ratis.local; + +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import picocli.CommandLine; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Handler to generate a new raft-meta.conf + */ +@CommandLine.Command( + name = "raftMetaConf", + description = "Generates a new raft-meta.conf", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class +) + +public class RatisLocalRaftMetaConfSubCommand extends ScmSubcommand { + + @CommandLine.Option(names = { "-peers" }, + description = "Provide list of peers in format" + + " <[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>", + required = true) + private String peers; + + @CommandLine.Option(names = { "-path" }, + description = "Parent path of raft-meta.conf", + required = true) + private String path; + + @Override + public void execute(ScmClient scmClient) throws IOException { + if (peers == null || path == null || peers.isEmpty() || path.isEmpty()) { + System.err.println("peers or path cannot be empty."); + return; + } + Set addresses = new HashSet<>(); + Set ids = new HashSet<>(); + List raftPeerProtos = new ArrayList<>(); + String[] peerArray = peers.split(","); + + for (String idWithAddress : peerArray) { + String[] peerIdWithAddressArray = idWithAddress.split("\\|"); + + if (peerIdWithAddressArray.length < 1 || peerIdWithAddressArray.length > 2) { + System.err.println( + "Failed to parse peer's ID and address for: " + idWithAddress + ", from option: -peers " + peers + + ". Please provide list of peers in format " + + "<[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>"); + return; + } + + InetSocketAddress inetSocketAddress = parseInetSocketAddress( + peerIdWithAddressArray[peerIdWithAddressArray.length - 1]); + String addressString = inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort(); + + if (addresses.contains(addressString)) { + System.err.println("Found duplicated address: " + addressString + + ". Please ensure the address of peers have no duplicated value."); + return; + } + addresses.add(addressString); + + String peerId; + if (peerIdWithAddressArray.length == 2) { + peerId = RaftPeerId.getRaftPeerId(peerIdWithAddressArray[0]).toString(); + + if (ids.contains(peerId)) { + System.err.println("Found duplicated ID: " + peerId + + ". Please ensure the ID of peers have no duplicated value."); + return; + } + ids.add(peerId); + } else { + peerId = getPeerId(inetSocketAddress).toString(); + } + + raftPeerProtos.add(RaftProtos.RaftPeerProto.newBuilder() + .setId(ByteString.copyFrom(peerId.getBytes(StandardCharsets.UTF_8))) + .setAddress(addressString) + .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) + .build()); + } + + try (InputStream in = Files.newInputStream(Paths.get(path, "raft-meta.conf")); + OutputStream out = Files.newOutputStream(Paths.get(path, "new-raft-meta.conf"))) { + + long index = RaftProtos.LogEntryProto.newBuilder().mergeFrom(in).build().getIndex(); + System.out.println("Index in the original file is: " + index); + + RaftProtos.LogEntryProto newLogEntryProto = RaftProtos.LogEntryProto.newBuilder() + .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() + .addAllPeers(raftPeerProtos).build()) + .setIndex(index + 1) + .build(); + + System.out.println("Generated new LogEntryProto info:\n" + newLogEntryProto); + newLogEntryProto.writeTo(out); + } + } + + public static InetSocketAddress parseInetSocketAddress(String address) { + try { + final String[] hostPortPair = address.split(":"); + if (hostPortPair.length < 2) { + throw new IllegalArgumentException("Unexpected address format ."); + } + return new InetSocketAddress(hostPortPair[0], Integer.parseInt(hostPortPair[1])); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse the server address parameter \"" + address + "\".", e); + } + } + + public static RaftPeerId getPeerId(InetSocketAddress address) { + return getPeerId(address.getHostString(), address.getPort()); + } + + public static RaftPeerId getPeerId(String host, int port) { + return RaftPeerId.getRaftPeerId(host + "_" + port); + } +} diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalSubCommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalSubCommand.java new file mode 100644 index 000000000000..3dfd7d4999ad --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalSubCommand.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.ratis.local; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.apache.hadoop.hdds.scm.cli.ratis.RatisCommands; +import org.kohsuke.MetaInfServices; +import picocli.CommandLine; + +import java.util.concurrent.Callable; + +/** + * Subcommand for ratis local related operations. + */ +@CommandLine.Command( + name = "local", + description = "Local operations within Ratis", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class, + subcommands = { + RatisLocalRaftMetaConfSubCommand.class + } +) +@MetaInfServices(SubcommandWithParent.class) +public class RatisLocalSubCommand implements Callable, SubcommandWithParent { + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public Void call() throws Exception { + GenericCli.missingSubcommand(spec); + return null; + } + + @Override + public Class getParentType() { + return RatisCommands.class; + } +} diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java new file mode 100644 index 000000000000..1f04b954b2ff --- /dev/null +++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.ratis; + +import org.apache.hadoop.hdds.scm.cli.ratis.local.RatisLocalRaftMetaConfSubCommand; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import picocli.CommandLine; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.InputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +/** + * Tests for RatisLocalRaftMetaConfSubCommand class. + */ +public class TestRatisLocalRaftMetaConfSubCommand { + + private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name(); + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + private RatisLocalRaftMetaConfSubCommand raftCmd; + private ScmClient scmClient; + + @BeforeEach + public void setup() throws IOException { + raftCmd = new RatisLocalRaftMetaConfSubCommand(); + System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING)); + System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING)); + + scmClient = mock(ScmClient.class); + } + + @AfterEach + public void tearDown() { + System.setOut(originalOut); + System.setErr(originalErr); + } + + @Test + public void testLocalRaftMetaConfSubcommand(@TempDir Path tempDir) throws IOException { + // Set up temporary directory and files + Path metadataDir = tempDir.resolve("data/metadata/ratis/test-cluster/current/"); + Files.createDirectories(metadataDir); + + // Create a dummy raft-meta.conf file using protobuf + Path raftMetaConfFile = metadataDir.resolve("raft-meta.conf"); + + // Create a LogEntryProto with a dummy index and peer + RaftProtos.RaftPeerProto raftPeerProto = RaftProtos.RaftPeerProto.newBuilder() + .setId(ByteString.copyFromUtf8("peer1")) + .setAddress("localhost:8000") + .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) + .build(); + + RaftProtos.LogEntryProto logEntryProto = RaftProtos.LogEntryProto.newBuilder() + .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() + .addPeers(raftPeerProto).build()) + .setIndex(1) + .build(); + + // Write the logEntryProto to the raft-meta.conf file + try (OutputStream out = Files.newOutputStream(raftMetaConfFile)) { + logEntryProto.writeTo(out); + } + + CommandLine cmd = new CommandLine(raftCmd); + cmd.parseArgs("-peers", "peer1|localhost:8080", "-path", metadataDir.toString()); + + raftCmd.execute(scmClient); + + String output = outContent.toString(DEFAULT_ENCODING); + assertTrue(output.contains("Index in the original file is: 1")); + assertTrue(output.contains("Generated new LogEntryProto info")); + + // Verify that the new raft-meta.conf is generated + Path newRaftMetaConfFile = metadataDir.resolve("new-raft-meta.conf"); + assertTrue(Files.exists(newRaftMetaConfFile)); + + // Verify content of the newly generated file + try (InputStream in = Files.newInputStream(newRaftMetaConfFile)) { + RaftProtos.LogEntryProto newLogEntryProto = RaftProtos.LogEntryProto.parseFrom(in); + assertEquals(2, newLogEntryProto.getIndex()); + RaftProtos.RaftPeerProto peerProto = newLogEntryProto.getConfigurationEntry().getPeers(0); + assertEquals("peer1", peerProto.getId().toStringUtf8()); + assertEquals("localhost:8080", peerProto.getAddress()); + assertEquals(RaftProtos.RaftPeerRole.FOLLOWER, peerProto.getStartupRole()); + } + } + + @Test + public void testMissingRequiredArguments() { + CommandLine cmd = new CommandLine(raftCmd); + + Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { + cmd.parseArgs(); + raftCmd.execute(scmClient); + }); + + assertEquals("Missing required options: '-peers=', '-path='", exception.getMessage()); + } + + @Test + public void testMissingPeersArgument() { + CommandLine cmd = new CommandLine(raftCmd); + + Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { + cmd.parseArgs("-path", "/dummy/path"); + raftCmd.execute(scmClient); + }); + + assertEquals("Missing required option: '-peers='", exception.getMessage()); + } + + @Test + public void testMissingPathArgument() { + CommandLine cmd = new CommandLine(raftCmd); + + Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { + cmd.parseArgs("-peers", "peer1|localhost:8080"); + raftCmd.execute(scmClient); + }); + + assertEquals("Missing required option: '-path='", exception.getMessage()); + } + + @Test + public void testInvalidPeersFormat() { + CommandLine cmd = new CommandLine(raftCmd); + + IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> { + cmd.parseArgs("-peers", "peer1|localhost8080", "-path", "/dummy/path"); //invalid peers format (missing ':' separator) + raftCmd.execute(scmClient); + }); + + assertTrue(thrown.getMessage().contains("Failed to parse the server address parameter")); + } + + @Test + public void testDuplicatePeersAddress() throws IOException { + CommandLine cmd = new CommandLine(raftCmd); + cmd.parseArgs("-peers", "localhost:8080,localhost:8080", "-path", "/dummy/path"); + raftCmd.execute(scmClient); + + String errorOutput = errContent.toString(DEFAULT_ENCODING); + assertTrue(errorOutput.contains("Found duplicated address")); + } + + @Test + public void testDuplicatePeersId() throws IOException { + CommandLine cmd = new CommandLine(raftCmd); + cmd.parseArgs("-peers", "peer1|localhost:8080,peer1|localhost:8081", "-path", "/dummy/path"); + raftCmd.execute(scmClient); + + String errorOutput = errContent.toString(DEFAULT_ENCODING); + assertTrue(errorOutput.contains("Found duplicated ID")); + } +} From abbafe517c36723c7cc00d6e4f3d45e4169ab76a Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Fri, 6 Sep 2024 12:12:22 +0530 Subject: [PATCH 02/10] Fixed checkstyle errors --- .../RatisLocalRaftMetaConfSubCommand.java | 40 +++++++++---------- .../scm/cli/ratis/local/package-info.java | 22 ++++++++++ .../hdds/scm/cli/ratis/package-info.java | 22 ++++++++++ .../TestRatisLocalRaftMetaConfSubCommand.java | 23 ++++++----- 4 files changed, 77 insertions(+), 30 deletions(-) create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/package-info.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/package-info.java diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java index f2a4c0890331..f0d085358541 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java @@ -38,13 +38,13 @@ import java.util.Set; /** - * Handler to generate a new raft-meta.conf + * Handler to generate a new raft-meta.conf. */ @CommandLine.Command( - name = "raftMetaConf", - description = "Generates a new raft-meta.conf", - mixinStandardHelpOptions = true, - versionProvider = HddsVersionProvider.class + name = "raftMetaConf", + description = "Generates a new raft-meta.conf", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class ) public class RatisLocalRaftMetaConfSubCommand extends ScmSubcommand { @@ -76,19 +76,19 @@ public void execute(ScmClient scmClient) throws IOException { if (peerIdWithAddressArray.length < 1 || peerIdWithAddressArray.length > 2) { System.err.println( - "Failed to parse peer's ID and address for: " + idWithAddress + ", from option: -peers " + peers + - ". Please provide list of peers in format " + - "<[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>"); + "Failed to parse peer's ID and address for: " + idWithAddress + ", from option: -peers " + peers + + ". Please provide list of peers in format " + + "<[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>"); return; } InetSocketAddress inetSocketAddress = parseInetSocketAddress( - peerIdWithAddressArray[peerIdWithAddressArray.length - 1]); + peerIdWithAddressArray[peerIdWithAddressArray.length - 1]); String addressString = inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort(); if (addresses.contains(addressString)) { System.err.println("Found duplicated address: " + addressString + - ". Please ensure the address of peers have no duplicated value."); + ". Please ensure the address of peers have no duplicated value."); return; } addresses.add(addressString); @@ -99,7 +99,7 @@ public void execute(ScmClient scmClient) throws IOException { if (ids.contains(peerId)) { System.err.println("Found duplicated ID: " + peerId + - ". Please ensure the ID of peers have no duplicated value."); + ". Please ensure the ID of peers have no duplicated value."); return; } ids.add(peerId); @@ -108,23 +108,23 @@ public void execute(ScmClient scmClient) throws IOException { } raftPeerProtos.add(RaftProtos.RaftPeerProto.newBuilder() - .setId(ByteString.copyFrom(peerId.getBytes(StandardCharsets.UTF_8))) - .setAddress(addressString) - .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) - .build()); + .setId(ByteString.copyFrom(peerId.getBytes(StandardCharsets.UTF_8))) + .setAddress(addressString) + .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) + .build()); } try (InputStream in = Files.newInputStream(Paths.get(path, "raft-meta.conf")); - OutputStream out = Files.newOutputStream(Paths.get(path, "new-raft-meta.conf"))) { + OutputStream out = Files.newOutputStream(Paths.get(path, "new-raft-meta.conf"))) { long index = RaftProtos.LogEntryProto.newBuilder().mergeFrom(in).build().getIndex(); System.out.println("Index in the original file is: " + index); RaftProtos.LogEntryProto newLogEntryProto = RaftProtos.LogEntryProto.newBuilder() - .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() - .addAllPeers(raftPeerProtos).build()) - .setIndex(index + 1) - .build(); + .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() + .addAllPeers(raftPeerProtos).build()) + .setIndex(index + 1) + .build(); System.out.println("Generated new LogEntryProto info:\n" + newLogEntryProto); newLogEntryProto.writeTo(out); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/package-info.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/package-info.java new file mode 100644 index 000000000000..43ee29e9d225 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Contains all of the ratis local related commands. + */ +package org.apache.hadoop.hdds.scm.cli.ratis.local; diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/package-info.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/package-info.java new file mode 100644 index 000000000000..6f44ed856af2 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Contains all of the ratis related scm commands. + */ +package org.apache.hadoop.hdds.scm.cli.ratis; diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java index 1f04b954b2ff..5d9996ef1531 100644 --- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java +++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java @@ -36,7 +36,9 @@ import java.nio.file.Files; import java.nio.file.Path; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; /** @@ -78,16 +80,16 @@ public void testLocalRaftMetaConfSubcommand(@TempDir Path tempDir) throws IOExce // Create a LogEntryProto with a dummy index and peer RaftProtos.RaftPeerProto raftPeerProto = RaftProtos.RaftPeerProto.newBuilder() - .setId(ByteString.copyFromUtf8("peer1")) - .setAddress("localhost:8000") - .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) - .build(); + .setId(ByteString.copyFromUtf8("peer1")) + .setAddress("localhost:8000") + .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) + .build(); RaftProtos.LogEntryProto logEntryProto = RaftProtos.LogEntryProto.newBuilder() - .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() - .addPeers(raftPeerProto).build()) - .setIndex(1) - .build(); + .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() + .addPeers(raftPeerProto).build()) + .setIndex(1) + .build(); // Write the logEntryProto to the raft-meta.conf file try (OutputStream out = Files.newOutputStream(raftMetaConfFile)) { @@ -159,7 +161,8 @@ public void testInvalidPeersFormat() { CommandLine cmd = new CommandLine(raftCmd); IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> { - cmd.parseArgs("-peers", "peer1|localhost8080", "-path", "/dummy/path"); //invalid peers format (missing ':' separator) + //invalid peers format (missing ':' separator) + cmd.parseArgs("-peers", "peer1|localhost8080", "-path", "/dummy/path"); raftCmd.execute(scmClient); }); From b2820f483e00363c5301087a3a2e9980a6d58fe8 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Fri, 6 Sep 2024 17:29:16 +0530 Subject: [PATCH 03/10] Moved the files --- .../ozone/admin}/ratis/RatisCommands.java | 2 +- .../admin/ratis/local/RatisLocalCommand.java | 6 ++--- .../RatisLocalRaftMetaConfSubCommand.java | 18 +++++++-------- .../admin/ratis/local}/package-info.java | 10 ++++---- .../ozone/admin/ratis}/package-info.java | 10 ++++---- .../TestRatisLocalRaftMetaConfSubCommand.java | 23 ++++++++----------- .../ozone/admin/ratis/package-info.java | 23 +++++++++++++++++++ 7 files changed, 55 insertions(+), 37 deletions(-) rename {hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin}/ratis/RatisCommands.java (97%) rename hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalSubCommand.java => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalCommand.java (89%) rename {hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin}/ratis/local/RatisLocalRaftMetaConfSubCommand.java (93%) rename {hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local}/package-info.java (83%) rename {hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis}/package-info.java (82%) rename {hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli => hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin}/ratis/TestRatisLocalRaftMetaConfSubCommand.java (92%) create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/package-info.java diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/RatisCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/RatisCommands.java similarity index 97% rename from hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/RatisCommands.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/RatisCommands.java index b9918af65ba5..8bb8a0adced1 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/RatisCommands.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/RatisCommands.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdds.scm.cli.ratis; +package org.apache.hadoop.ozone.admin.ratis; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalCommand.java similarity index 89% rename from hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalSubCommand.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalCommand.java index 3dfd7d4999ad..57654c472883 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalCommand.java @@ -15,12 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdds.scm.cli.ratis.local; +package org.apache.hadoop.ozone.admin.ratis.local; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.cli.SubcommandWithParent; -import org.apache.hadoop.hdds.scm.cli.ratis.RatisCommands; +import org.apache.hadoop.ozone.admin.ratis.RatisCommands; import org.kohsuke.MetaInfServices; import picocli.CommandLine; @@ -39,7 +39,7 @@ } ) @MetaInfServices(SubcommandWithParent.class) -public class RatisLocalSubCommand implements Callable, SubcommandWithParent { +public class RatisLocalCommand implements Callable, SubcommandWithParent { @CommandLine.Spec private CommandLine.Model.CommandSpec spec; diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalRaftMetaConfSubCommand.java similarity index 93% rename from hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalRaftMetaConfSubCommand.java index f0d085358541..33381a3bf576 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/RatisLocalRaftMetaConfSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalRaftMetaConfSubCommand.java @@ -15,11 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdds.scm.cli.ratis.local; +package org.apache.hadoop.ozone.admin.ratis.local; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; -import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.ratis.proto.RaftProtos; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; @@ -36,6 +34,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.Callable; /** * Handler to generate a new raft-meta.conf. @@ -47,7 +46,7 @@ versionProvider = HddsVersionProvider.class ) -public class RatisLocalRaftMetaConfSubCommand extends ScmSubcommand { +public class RatisLocalRaftMetaConfSubCommand implements Callable { @CommandLine.Option(names = { "-peers" }, description = "Provide list of peers in format" + @@ -61,10 +60,10 @@ public class RatisLocalRaftMetaConfSubCommand extends ScmSubcommand { private String path; @Override - public void execute(ScmClient scmClient) throws IOException { + public Void call() throws IOException { if (peers == null || path == null || peers.isEmpty() || path.isEmpty()) { System.err.println("peers or path cannot be empty."); - return; + return null; } Set addresses = new HashSet<>(); Set ids = new HashSet<>(); @@ -79,7 +78,7 @@ public void execute(ScmClient scmClient) throws IOException { "Failed to parse peer's ID and address for: " + idWithAddress + ", from option: -peers " + peers + ". Please provide list of peers in format " + "<[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>"); - return; + return null; } InetSocketAddress inetSocketAddress = parseInetSocketAddress( @@ -89,7 +88,7 @@ public void execute(ScmClient scmClient) throws IOException { if (addresses.contains(addressString)) { System.err.println("Found duplicated address: " + addressString + ". Please ensure the address of peers have no duplicated value."); - return; + return null; } addresses.add(addressString); @@ -100,7 +99,7 @@ public void execute(ScmClient scmClient) throws IOException { if (ids.contains(peerId)) { System.err.println("Found duplicated ID: " + peerId + ". Please ensure the ID of peers have no duplicated value."); - return; + return null; } ids.add(peerId); } else { @@ -129,6 +128,7 @@ public void execute(ScmClient scmClient) throws IOException { System.out.println("Generated new LogEntryProto info:\n" + newLogEntryProto); newLogEntryProto.writeTo(out); } + return null; } public static InetSocketAddress parseInetSocketAddress(String address) { diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java similarity index 83% rename from hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/package-info.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java index 6f44ed856af2..183a1f8e97a5 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,6 @@ */ /** - * Contains all of the ratis related scm commands. + * Ratis Local related tools */ -package org.apache.hadoop.hdds.scm.cli.ratis; +package org.apache.hadoop.ozone.admin.ratis.local; diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java similarity index 82% rename from hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/package-info.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java index 43ee29e9d225..e6401654d741 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ratis/local/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,6 @@ */ /** - * Contains all of the ratis local related commands. + * Ratis related Admin Tools */ -package org.apache.hadoop.hdds.scm.cli.ratis.local; +package org.apache.hadoop.ozone.admin.ratis; diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/TestRatisLocalRaftMetaConfSubCommand.java similarity index 92% rename from hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/TestRatisLocalRaftMetaConfSubCommand.java index 5d9996ef1531..345e7cb7e199 100644 --- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/ratis/TestRatisLocalRaftMetaConfSubCommand.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/TestRatisLocalRaftMetaConfSubCommand.java @@ -15,10 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdds.scm.cli.ratis; +package org.apache.hadoop.ozone.admin.ratis; -import org.apache.hadoop.hdds.scm.cli.ratis.local.RatisLocalRaftMetaConfSubCommand; -import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.ozone.admin.ratis.local.RatisLocalRaftMetaConfSubCommand; import org.apache.ratis.proto.RaftProtos; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterEach; @@ -39,7 +38,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; /** * Tests for RatisLocalRaftMetaConfSubCommand class. @@ -52,15 +50,12 @@ public class TestRatisLocalRaftMetaConfSubCommand { private final PrintStream originalOut = System.out; private final PrintStream originalErr = System.err; private RatisLocalRaftMetaConfSubCommand raftCmd; - private ScmClient scmClient; @BeforeEach public void setup() throws IOException { raftCmd = new RatisLocalRaftMetaConfSubCommand(); System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING)); System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING)); - - scmClient = mock(ScmClient.class); } @AfterEach @@ -99,7 +94,7 @@ public void testLocalRaftMetaConfSubcommand(@TempDir Path tempDir) throws IOExce CommandLine cmd = new CommandLine(raftCmd); cmd.parseArgs("-peers", "peer1|localhost:8080", "-path", metadataDir.toString()); - raftCmd.execute(scmClient); + raftCmd.call(); String output = outContent.toString(DEFAULT_ENCODING); assertTrue(output.contains("Index in the original file is: 1")); @@ -126,7 +121,7 @@ public void testMissingRequiredArguments() { Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { cmd.parseArgs(); - raftCmd.execute(scmClient); + raftCmd.call(); }); assertEquals("Missing required options: '-peers=', '-path='", exception.getMessage()); @@ -138,7 +133,7 @@ public void testMissingPeersArgument() { Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { cmd.parseArgs("-path", "/dummy/path"); - raftCmd.execute(scmClient); + raftCmd.call(); }); assertEquals("Missing required option: '-peers='", exception.getMessage()); @@ -150,7 +145,7 @@ public void testMissingPathArgument() { Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { cmd.parseArgs("-peers", "peer1|localhost:8080"); - raftCmd.execute(scmClient); + raftCmd.call(); }); assertEquals("Missing required option: '-path='", exception.getMessage()); @@ -163,7 +158,7 @@ public void testInvalidPeersFormat() { IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> { //invalid peers format (missing ':' separator) cmd.parseArgs("-peers", "peer1|localhost8080", "-path", "/dummy/path"); - raftCmd.execute(scmClient); + raftCmd.call(); }); assertTrue(thrown.getMessage().contains("Failed to parse the server address parameter")); @@ -173,7 +168,7 @@ public void testInvalidPeersFormat() { public void testDuplicatePeersAddress() throws IOException { CommandLine cmd = new CommandLine(raftCmd); cmd.parseArgs("-peers", "localhost:8080,localhost:8080", "-path", "/dummy/path"); - raftCmd.execute(scmClient); + raftCmd.call(); String errorOutput = errContent.toString(DEFAULT_ENCODING); assertTrue(errorOutput.contains("Found duplicated address")); @@ -183,7 +178,7 @@ public void testDuplicatePeersAddress() throws IOException { public void testDuplicatePeersId() throws IOException { CommandLine cmd = new CommandLine(raftCmd); cmd.parseArgs("-peers", "peer1|localhost:8080,peer1|localhost:8081", "-path", "/dummy/path"); - raftCmd.execute(scmClient); + raftCmd.call(); String errorOutput = errContent.toString(DEFAULT_ENCODING); assertTrue(errorOutput.contains("Found duplicated ID")); diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/package-info.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/package-info.java new file mode 100644 index 000000000000..46ad468b6c03 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Tests for Ratis Commands. + */ +package org.apache.hadoop.ozone.admin.ratis; + From a738bf05facf273b1f0014e9b71c8f255ea878ed Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Fri, 6 Sep 2024 17:36:02 +0530 Subject: [PATCH 04/10] Fixed checkstyle errors - 2 --- .../org/apache/hadoop/ozone/admin/ratis/local/package-info.java | 2 +- .../java/org/apache/hadoop/ozone/admin/ratis/package-info.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java index 183a1f8e97a5..719b2a126e89 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java @@ -17,6 +17,6 @@ */ /** - * Ratis Local related tools + * Ratis Local related tools. */ package org.apache.hadoop.ozone.admin.ratis.local; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java index e6401654d741..e83b40c2dc9e 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java @@ -17,6 +17,6 @@ */ /** - * Ratis related Admin Tools + * Ratis related Admin Tools. */ package org.apache.hadoop.ozone.admin.ratis; From 9100b6059857e608f7d2ec7f0bb5b75cfef0d7b1 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Thu, 19 Sep 2024 08:51:06 +0530 Subject: [PATCH 05/10] Support ozone to forward arguments to Ratis --- hadoop-ozone/dist/src/shell/ozone/ozone | 5 + hadoop-ozone/tools/pom.xml | 15 ++ .../ozone/admin/ratis/RatisCommands.java | 54 ----- .../admin/ratis/local/RatisLocalCommand.java | 57 ------ .../RatisLocalRaftMetaConfSubCommand.java | 153 -------------- .../ozone/admin/ratis/local/package-info.java | 22 --- .../apache/hadoop/ozone/ratis/OzoneRatis.java | 58 ++++++ .../ozone/{admin => }/ratis/package-info.java | 4 +- .../TestRatisLocalRaftMetaConfSubCommand.java | 186 ------------------ .../hadoop/ozone/ratis/TestOzoneRatis.java | 159 +++++++++++++++ .../ozone/{admin => }/ratis/package-info.java | 5 +- 11 files changed, 241 insertions(+), 477 deletions(-) delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/RatisCommands.java delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalCommand.java delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalRaftMetaConfSubCommand.java delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/OzoneRatis.java rename hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{admin => }/ratis/package-info.java (91%) delete mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/TestRatisLocalRaftMetaConfSubCommand.java create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java rename hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/{admin => }/ratis/package-info.java (91%) diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 22ceed9ed3c6..9e9d70bcb45a 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -61,6 +61,7 @@ function ozone_usage ozone_add_subcommand "debug" client "Ozone debug tool" ozone_add_subcommand "repair" client "Ozone repair tool" ozone_add_subcommand "checknative" client "checks if native libraries are loaded" + ozone_add_subcommand "ratis" client "Ozone ratis tool" ozone_generate_usage "${OZONE_SHELL_EXECNAME}" false } @@ -231,6 +232,10 @@ function ozonecmd_case OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.checknative.CheckNative OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; + ratis) + OZONE_CLASSNAME=org.apache.hadoop.ozone.ratis.OzoneRatis + OZONE_RUN_ARTIFACT_NAME="ozone-tools" + ;; *) OZONE_CLASSNAME="${subcmd}" if ! ozone_validate_classname "${OZONE_CLASSNAME}"; then diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 04c1c8602cb0..04dc60e397dc 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -160,6 +160,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.ratis ratis-tools + + org.apache.ratis + ratis-shell + info.picocli @@ -249,6 +253,17 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hdds-test-utils test + + org.apache.ratis + ratis-shell + 3.1.0 + + + com.jamesmurty.utils + java-xmlbuilder + 0.4 + test + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/RatisCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/RatisCommands.java deleted file mode 100644 index 8bb8a0adced1..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/RatisCommands.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.admin.ratis; - -import org.apache.hadoop.hdds.cli.GenericCli; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.cli.OzoneAdmin; -import org.apache.hadoop.hdds.cli.SubcommandWithParent; -import org.kohsuke.MetaInfServices; -import picocli.CommandLine; - -import java.util.concurrent.Callable; - -/** - * Subcommand for ratis related operations. - */ -@CommandLine.Command( - name = "ratis", - description = "Ratis specific operations", - mixinStandardHelpOptions = true, - versionProvider = HddsVersionProvider.class -) -@MetaInfServices(SubcommandWithParent.class) -public class RatisCommands implements Callable, SubcommandWithParent { - - @CommandLine.Spec - private CommandLine.Model.CommandSpec spec; - - @Override - public Void call() throws Exception { - GenericCli.missingSubcommand(spec); - return null; - } - - @Override - public Class getParentType() { - return OzoneAdmin.class; - } -} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalCommand.java deleted file mode 100644 index 57654c472883..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalCommand.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.admin.ratis.local; - -import org.apache.hadoop.hdds.cli.GenericCli; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.cli.SubcommandWithParent; -import org.apache.hadoop.ozone.admin.ratis.RatisCommands; -import org.kohsuke.MetaInfServices; -import picocli.CommandLine; - -import java.util.concurrent.Callable; - -/** - * Subcommand for ratis local related operations. - */ -@CommandLine.Command( - name = "local", - description = "Local operations within Ratis", - mixinStandardHelpOptions = true, - versionProvider = HddsVersionProvider.class, - subcommands = { - RatisLocalRaftMetaConfSubCommand.class - } -) -@MetaInfServices(SubcommandWithParent.class) -public class RatisLocalCommand implements Callable, SubcommandWithParent { - - @CommandLine.Spec - private CommandLine.Model.CommandSpec spec; - - @Override - public Void call() throws Exception { - GenericCli.missingSubcommand(spec); - return null; - } - - @Override - public Class getParentType() { - return RatisCommands.class; - } -} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalRaftMetaConfSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalRaftMetaConfSubCommand.java deleted file mode 100644 index 33381a3bf576..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/RatisLocalRaftMetaConfSubCommand.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.admin.ratis.local; - -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.ratis.proto.RaftProtos; -import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; -import picocli.CommandLine; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.Callable; - -/** - * Handler to generate a new raft-meta.conf. - */ -@CommandLine.Command( - name = "raftMetaConf", - description = "Generates a new raft-meta.conf", - mixinStandardHelpOptions = true, - versionProvider = HddsVersionProvider.class -) - -public class RatisLocalRaftMetaConfSubCommand implements Callable { - - @CommandLine.Option(names = { "-peers" }, - description = "Provide list of peers in format" + - " <[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>", - required = true) - private String peers; - - @CommandLine.Option(names = { "-path" }, - description = "Parent path of raft-meta.conf", - required = true) - private String path; - - @Override - public Void call() throws IOException { - if (peers == null || path == null || peers.isEmpty() || path.isEmpty()) { - System.err.println("peers or path cannot be empty."); - return null; - } - Set addresses = new HashSet<>(); - Set ids = new HashSet<>(); - List raftPeerProtos = new ArrayList<>(); - String[] peerArray = peers.split(","); - - for (String idWithAddress : peerArray) { - String[] peerIdWithAddressArray = idWithAddress.split("\\|"); - - if (peerIdWithAddressArray.length < 1 || peerIdWithAddressArray.length > 2) { - System.err.println( - "Failed to parse peer's ID and address for: " + idWithAddress + ", from option: -peers " + peers + - ". Please provide list of peers in format " + - "<[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>"); - return null; - } - - InetSocketAddress inetSocketAddress = parseInetSocketAddress( - peerIdWithAddressArray[peerIdWithAddressArray.length - 1]); - String addressString = inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort(); - - if (addresses.contains(addressString)) { - System.err.println("Found duplicated address: " + addressString + - ". Please ensure the address of peers have no duplicated value."); - return null; - } - addresses.add(addressString); - - String peerId; - if (peerIdWithAddressArray.length == 2) { - peerId = RaftPeerId.getRaftPeerId(peerIdWithAddressArray[0]).toString(); - - if (ids.contains(peerId)) { - System.err.println("Found duplicated ID: " + peerId + - ". Please ensure the ID of peers have no duplicated value."); - return null; - } - ids.add(peerId); - } else { - peerId = getPeerId(inetSocketAddress).toString(); - } - - raftPeerProtos.add(RaftProtos.RaftPeerProto.newBuilder() - .setId(ByteString.copyFrom(peerId.getBytes(StandardCharsets.UTF_8))) - .setAddress(addressString) - .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) - .build()); - } - - try (InputStream in = Files.newInputStream(Paths.get(path, "raft-meta.conf")); - OutputStream out = Files.newOutputStream(Paths.get(path, "new-raft-meta.conf"))) { - - long index = RaftProtos.LogEntryProto.newBuilder().mergeFrom(in).build().getIndex(); - System.out.println("Index in the original file is: " + index); - - RaftProtos.LogEntryProto newLogEntryProto = RaftProtos.LogEntryProto.newBuilder() - .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() - .addAllPeers(raftPeerProtos).build()) - .setIndex(index + 1) - .build(); - - System.out.println("Generated new LogEntryProto info:\n" + newLogEntryProto); - newLogEntryProto.writeTo(out); - } - return null; - } - - public static InetSocketAddress parseInetSocketAddress(String address) { - try { - final String[] hostPortPair = address.split(":"); - if (hostPortPair.length < 2) { - throw new IllegalArgumentException("Unexpected address format ."); - } - return new InetSocketAddress(hostPortPair[0], Integer.parseInt(hostPortPair[1])); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to parse the server address parameter \"" + address + "\".", e); - } - } - - public static RaftPeerId getPeerId(InetSocketAddress address) { - return getPeerId(address.getHostString(), address.getPort()); - } - - public static RaftPeerId getPeerId(String host, int port) { - return RaftPeerId.getRaftPeerId(host + "_" + port); - } -} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java deleted file mode 100644 index 719b2a126e89..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/local/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Ratis Local related tools. - */ -package org.apache.hadoop.ozone.admin.ratis.local; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/OzoneRatis.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/OzoneRatis.java new file mode 100644 index 000000000000..268ee2b0e120 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/OzoneRatis.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ratis; + +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.apache.hadoop.ozone.shell.Shell; +import org.apache.ratis.shell.cli.sh.RatisShell; + +import picocli.CommandLine; + +/** + * Ozone Ratis Command line tool. + */ +@CommandLine.Command(name = "ozone ratis", + description = "Shell for Ozone Ratis Operations", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true) +public class OzoneRatis extends Shell { + + public OzoneRatis() { + super(OzoneRatis.class); + } + + /** + * Main for the OzoneRatis Command handling. + * + * @param argv - System Args Strings[] + */ + public static void main(String[] argv) throws Exception { + new OzoneRatis().run(argv); + } + + @Override + public int execute(String[] argv) { + TracingUtil.initTracing("shell", createOzoneConfiguration()); + String spanName = "ozone ratis" + String.join(" ", argv); + return TracingUtil.executeInNewSpan(spanName, () -> { + final RatisShell shell = new RatisShell(System.out); + return shell.run(argv); + }); + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java similarity index 91% rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java index e83b40c2dc9e..57d2a507ab14 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/ratis/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java @@ -17,6 +17,6 @@ */ /** - * Ratis related Admin Tools. + * Ozone Ratis tool. */ -package org.apache.hadoop.ozone.admin.ratis; +package org.apache.hadoop.ozone.ratis; \ No newline at end of file diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/TestRatisLocalRaftMetaConfSubCommand.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/TestRatisLocalRaftMetaConfSubCommand.java deleted file mode 100644 index 345e7cb7e199..000000000000 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/TestRatisLocalRaftMetaConfSubCommand.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.admin.ratis; - -import org.apache.hadoop.ozone.admin.ratis.local.RatisLocalRaftMetaConfSubCommand; -import org.apache.ratis.proto.RaftProtos; -import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import picocli.CommandLine; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.InputStream; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * Tests for RatisLocalRaftMetaConfSubCommand class. - */ -public class TestRatisLocalRaftMetaConfSubCommand { - - private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name(); - private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); - private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); - private final PrintStream originalOut = System.out; - private final PrintStream originalErr = System.err; - private RatisLocalRaftMetaConfSubCommand raftCmd; - - @BeforeEach - public void setup() throws IOException { - raftCmd = new RatisLocalRaftMetaConfSubCommand(); - System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING)); - System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING)); - } - - @AfterEach - public void tearDown() { - System.setOut(originalOut); - System.setErr(originalErr); - } - - @Test - public void testLocalRaftMetaConfSubcommand(@TempDir Path tempDir) throws IOException { - // Set up temporary directory and files - Path metadataDir = tempDir.resolve("data/metadata/ratis/test-cluster/current/"); - Files.createDirectories(metadataDir); - - // Create a dummy raft-meta.conf file using protobuf - Path raftMetaConfFile = metadataDir.resolve("raft-meta.conf"); - - // Create a LogEntryProto with a dummy index and peer - RaftProtos.RaftPeerProto raftPeerProto = RaftProtos.RaftPeerProto.newBuilder() - .setId(ByteString.copyFromUtf8("peer1")) - .setAddress("localhost:8000") - .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) - .build(); - - RaftProtos.LogEntryProto logEntryProto = RaftProtos.LogEntryProto.newBuilder() - .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() - .addPeers(raftPeerProto).build()) - .setIndex(1) - .build(); - - // Write the logEntryProto to the raft-meta.conf file - try (OutputStream out = Files.newOutputStream(raftMetaConfFile)) { - logEntryProto.writeTo(out); - } - - CommandLine cmd = new CommandLine(raftCmd); - cmd.parseArgs("-peers", "peer1|localhost:8080", "-path", metadataDir.toString()); - - raftCmd.call(); - - String output = outContent.toString(DEFAULT_ENCODING); - assertTrue(output.contains("Index in the original file is: 1")); - assertTrue(output.contains("Generated new LogEntryProto info")); - - // Verify that the new raft-meta.conf is generated - Path newRaftMetaConfFile = metadataDir.resolve("new-raft-meta.conf"); - assertTrue(Files.exists(newRaftMetaConfFile)); - - // Verify content of the newly generated file - try (InputStream in = Files.newInputStream(newRaftMetaConfFile)) { - RaftProtos.LogEntryProto newLogEntryProto = RaftProtos.LogEntryProto.parseFrom(in); - assertEquals(2, newLogEntryProto.getIndex()); - RaftProtos.RaftPeerProto peerProto = newLogEntryProto.getConfigurationEntry().getPeers(0); - assertEquals("peer1", peerProto.getId().toStringUtf8()); - assertEquals("localhost:8080", peerProto.getAddress()); - assertEquals(RaftProtos.RaftPeerRole.FOLLOWER, peerProto.getStartupRole()); - } - } - - @Test - public void testMissingRequiredArguments() { - CommandLine cmd = new CommandLine(raftCmd); - - Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { - cmd.parseArgs(); - raftCmd.call(); - }); - - assertEquals("Missing required options: '-peers=', '-path='", exception.getMessage()); - } - - @Test - public void testMissingPeersArgument() { - CommandLine cmd = new CommandLine(raftCmd); - - Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { - cmd.parseArgs("-path", "/dummy/path"); - raftCmd.call(); - }); - - assertEquals("Missing required option: '-peers='", exception.getMessage()); - } - - @Test - public void testMissingPathArgument() { - CommandLine cmd = new CommandLine(raftCmd); - - Exception exception = assertThrows(CommandLine.MissingParameterException.class, () -> { - cmd.parseArgs("-peers", "peer1|localhost:8080"); - raftCmd.call(); - }); - - assertEquals("Missing required option: '-path='", exception.getMessage()); - } - - @Test - public void testInvalidPeersFormat() { - CommandLine cmd = new CommandLine(raftCmd); - - IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> { - //invalid peers format (missing ':' separator) - cmd.parseArgs("-peers", "peer1|localhost8080", "-path", "/dummy/path"); - raftCmd.call(); - }); - - assertTrue(thrown.getMessage().contains("Failed to parse the server address parameter")); - } - - @Test - public void testDuplicatePeersAddress() throws IOException { - CommandLine cmd = new CommandLine(raftCmd); - cmd.parseArgs("-peers", "localhost:8080,localhost:8080", "-path", "/dummy/path"); - raftCmd.call(); - - String errorOutput = errContent.toString(DEFAULT_ENCODING); - assertTrue(errorOutput.contains("Found duplicated address")); - } - - @Test - public void testDuplicatePeersId() throws IOException { - CommandLine cmd = new CommandLine(raftCmd); - cmd.parseArgs("-peers", "peer1|localhost:8080,peer1|localhost:8081", "-path", "/dummy/path"); - raftCmd.call(); - - String errorOutput = errContent.toString(DEFAULT_ENCODING); - assertTrue(errorOutput.contains("Found duplicated ID")); - } -} diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java new file mode 100644 index 000000000000..18813bef9572 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ratis; + +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for OzoneRatis. + */ +public class TestOzoneRatis { + private static final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + private static final PrintStream printStream = new PrintStream(outputStream); + private OzoneRatis ozoneRatis; + + @BeforeEach + public void setUp() { + System.setOut(printStream); + ozoneRatis = new OzoneRatis(); + } + + /** + * Execute method to invoke the OzoneRatis class and capture output. + * + * @param args command line arguments to pass + * @return the output from OzoneRatis + */ + private String execute(String[] args) { + ozoneRatis.execute(args); + return outputStream.toString(); + } + + @Test + public void testBasicOzoneRatisCommand() { + String[] args = {""}; + String output = execute(args); + assertTrue(output.contains("Usage: ratis sh [generic options]")); + } + + @Test + public void testLocalRaftMetaConfSubcommand(@TempDir Path tempDir) throws IOException { + // Set up temporary directory and files + Path metadataDir = tempDir.resolve("data/metadata/ratis/test-cluster/current/"); + Files.createDirectories(metadataDir); + + // Create a dummy raft-meta.conf file using protobuf + Path raftMetaConfFile = metadataDir.resolve("raft-meta.conf"); + + // Create a LogEntryProto with a dummy index and peer + RaftProtos.RaftPeerProto raftPeerProto = RaftProtos.RaftPeerProto.newBuilder() + .setId(ByteString.copyFromUtf8("peer1")) + .setAddress("localhost:8000") + .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER) + .build(); + + RaftProtos.LogEntryProto logEntryProto = RaftProtos.LogEntryProto.newBuilder() + .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder() + .addPeers(raftPeerProto).build()) + .setIndex(0) + .build(); + + // Write the logEntryProto to the raft-meta.conf file + try (OutputStream out = Files.newOutputStream(raftMetaConfFile)) { + logEntryProto.writeTo(out); + } + + + String[] args = {"local", "raftMetaConf", "-peers", "peer1|localhost:8080", "-path", metadataDir.toString()}; + String output = execute(args); + + assertTrue(output.contains("Index in the original file is: 0")); + assertTrue(output.contains("Generate new LogEntryProto info is:")); + + // Verify that the new raft-meta.conf is generated + Path newRaftMetaConfFile = metadataDir.resolve("new-raft-meta.conf"); + assertTrue(Files.exists(newRaftMetaConfFile), "New raft-meta.conf file should be created."); + + // Verify content of the newly generated file + try (InputStream in = Files.newInputStream(newRaftMetaConfFile)) { + RaftProtos.LogEntryProto newLogEntryProto = RaftProtos.LogEntryProto.parseFrom(in); + assertEquals(1, newLogEntryProto.getIndex()); + RaftProtos.RaftPeerProto peerProto = newLogEntryProto.getConfigurationEntry().getPeers(0); + assertEquals("peer1", peerProto.getId().toStringUtf8()); + assertEquals("localhost:8080", peerProto.getAddress()); + assertEquals(RaftProtos.RaftPeerRole.FOLLOWER, peerProto.getStartupRole()); + } + } + + @Test + public void testMissingRequiredArguments() { + String[] args = {"local", "raftMetaConf"}; + String output = execute(args); + assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required options: peers, path")); + } + + @Test + public void testMissingPeerArgument() { + String[] args = {"local", "raftMetaConf", "-path", "/path"}; + String output = execute(args); + assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required option: peers")); + } + + @Test + public void testMissingPathArgument() { + String[] args = {"local", "raftMetaConf", "-peers", "localhost:8080"}; + String output = execute(args); + assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required option: path")); + } + + @Test + public void testInvalidPeersFormat() { + String[] args = {"local", "raftMetaConf", "-peers", "localhost8080", "-path", "/path"}; + String output = execute(args); + assertTrue(output.contains("Failed to parse the server address parameter \"localhost8080\".")); + } + + @Test + public void testDuplicatePeersAddress() { + String[] args = {"local", "raftMetaConf", "-peers", "localhost:8080,localhost:8080", "-path", "/path"}; + String output = execute(args); + assertTrue(output.contains("Found duplicated address: localhost:8080.")); + } + + @Test + public void testDuplicatePeersId() { + String[] args = {"local", "raftMetaConf", "-peers", "peer1|localhost:8080,peer1|localhost:8081", "-path", "/path"}; + String output = execute(args); + assertTrue(output.contains("Found duplicated ID: peer1.")); + } +} diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/package-info.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java similarity index 91% rename from hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/package-info.java rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java index 46ad468b6c03..277db82e711a 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/admin/ratis/package-info.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java @@ -17,7 +17,6 @@ */ /** - * Tests for Ratis Commands. + * Tests for Ozone Ratis. */ -package org.apache.hadoop.ozone.admin.ratis; - +package org.apache.hadoop.ozone.ratis; \ No newline at end of file From e04cb2d0a80c5ffaf47a0c79db985e1f16950b89 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Thu, 19 Sep 2024 09:02:48 +0530 Subject: [PATCH 06/10] Fixed minor issues --- hadoop-ozone/tools/pom.xml | 11 ----------- .../org/apache/hadoop/ozone/ratis/package-info.java | 2 +- .../org/apache/hadoop/ozone/ratis/TestOzoneRatis.java | 4 ++-- .../org/apache/hadoop/ozone/ratis/package-info.java | 2 +- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 04dc60e397dc..d8c5599f3046 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -253,17 +253,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hdds-test-utils test - - org.apache.ratis - ratis-shell - 3.1.0 - - - com.jamesmurty.utils - java-xmlbuilder - 0.4 - test - diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java index 57d2a507ab14..a202f58438e5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java @@ -19,4 +19,4 @@ /** * Ozone Ratis tool. */ -package org.apache.hadoop.ozone.ratis; \ No newline at end of file +package org.apache.hadoop.ozone.ratis; diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java index 18813bef9572..1755093aa355 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java @@ -38,8 +38,8 @@ * Tests for OzoneRatis. */ public class TestOzoneRatis { - private static final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - private static final PrintStream printStream = new PrintStream(outputStream); + private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + private final PrintStream printStream = new PrintStream(outputStream); private OzoneRatis ozoneRatis; @BeforeEach diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java index 277db82e711a..c585018c5f1f 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java @@ -19,4 +19,4 @@ /** * Tests for Ozone Ratis. */ -package org.apache.hadoop.ozone.ratis; \ No newline at end of file +package org.apache.hadoop.ozone.ratis; From 6ce8591fcaa4b0062c563082e2b1b5d39bc8589d Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Thu, 19 Sep 2024 10:01:38 +0530 Subject: [PATCH 07/10] Added dependency version --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index 821cd911a202..dc4e2ad69332 100644 --- a/pom.xml +++ b/pom.xml @@ -758,6 +758,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs ratis-common ${ratis.version} + + org.apache.ratis + ratis-shell + ${ratis.version} + io.netty From b571f25828ce8d63544a61dbe074e062007a1af2 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Thu, 19 Sep 2024 12:47:09 +0530 Subject: [PATCH 08/10] Dependency issue fixe3d --- .../dist/src/main/license/bin/LICENSE.txt | 2 + .../dist/src/main/license/jar-report.txt | 2 + .../hadoop/ozone/ratis/TestOzoneRatis.java | 39 ++++++++++++------- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt index c28483c67356..9b846ca722ee 100644 --- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt @@ -407,6 +407,7 @@ Apache License 2.0 org.apache.ratis:ratis-proto org.apache.ratis:ratis-server org.apache.ratis:ratis-server-api + org.apache.ratis:ratis-shell org.apache.ratis:ratis-thirdparty-misc org.apache.ratis:ratis-tools org.apache.thrift:libthrift @@ -458,6 +459,7 @@ MIT org.kohsuke.metainf-services:metainf-services org.slf4j:slf4j-api org.slf4j:slf4j-reload4j + org.slf4j:slf4j-simple Public Domain diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 042c9380e4af..22139ae86e8e 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -252,6 +252,7 @@ share/ozone/lib/ratis-netty.jar share/ozone/lib/ratis-proto.jar share/ozone/lib/ratis-server-api.jar share/ozone/lib/ratis-server.jar +share/ozone/lib/ratis-shell.jar share/ozone/lib/ratis-thirdparty-misc.jar share/ozone/lib/ratis-tools.jar share/ozone/lib/re2j.jar @@ -264,6 +265,7 @@ share/ozone/lib/simpleclient_dropwizard.jar share/ozone/lib/simpleclient.jar share/ozone/lib/slf4j-api.jar share/ozone/lib/slf4j-reload4j.jar +share/ozone/lib/slf4j-simple.jar share/ozone/lib/snakeyaml.jar share/ozone/lib/snappy-java.jar share/ozone/lib/spring-beans.jar diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java index 1755093aa355..4d9e3acd3296 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java @@ -19,6 +19,7 @@ import org.apache.ratis.proto.RaftProtos; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -28,8 +29,10 @@ import java.io.OutputStream; import java.io.PrintStream; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.charset.StandardCharsets; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -38,29 +41,39 @@ * Tests for OzoneRatis. */ public class TestOzoneRatis { - private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - private final PrintStream printStream = new PrintStream(outputStream); + private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name(); + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; private OzoneRatis ozoneRatis; @BeforeEach - public void setUp() { - System.setOut(printStream); + public void setUp() throws UnsupportedEncodingException { + System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING)); + System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING)); ozoneRatis = new OzoneRatis(); } + @AfterEach + public void tearDown() { + System.setOut(originalOut); + System.setErr(originalErr); + } + /** * Execute method to invoke the OzoneRatis class and capture output. * * @param args command line arguments to pass * @return the output from OzoneRatis */ - private String execute(String[] args) { + private String execute(String[] args) throws IOException { ozoneRatis.execute(args); - return outputStream.toString(); + return outContent.toString(StandardCharsets.UTF_8.name()); } @Test - public void testBasicOzoneRatisCommand() { + public void testBasicOzoneRatisCommand() throws IOException { String[] args = {""}; String output = execute(args); assertTrue(output.contains("Usage: ratis sh [generic options]")); @@ -116,42 +129,42 @@ public void testLocalRaftMetaConfSubcommand(@TempDir Path tempDir) throws IOExce } @Test - public void testMissingRequiredArguments() { + public void testMissingRequiredArguments() throws IOException { String[] args = {"local", "raftMetaConf"}; String output = execute(args); assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required options: peers, path")); } @Test - public void testMissingPeerArgument() { + public void testMissingPeerArgument() throws IOException { String[] args = {"local", "raftMetaConf", "-path", "/path"}; String output = execute(args); assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required option: peers")); } @Test - public void testMissingPathArgument() { + public void testMissingPathArgument() throws IOException { String[] args = {"local", "raftMetaConf", "-peers", "localhost:8080"}; String output = execute(args); assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required option: path")); } @Test - public void testInvalidPeersFormat() { + public void testInvalidPeersFormat() throws IOException { String[] args = {"local", "raftMetaConf", "-peers", "localhost8080", "-path", "/path"}; String output = execute(args); assertTrue(output.contains("Failed to parse the server address parameter \"localhost8080\".")); } @Test - public void testDuplicatePeersAddress() { + public void testDuplicatePeersAddress() throws IOException { String[] args = {"local", "raftMetaConf", "-peers", "localhost:8080,localhost:8080", "-path", "/path"}; String output = execute(args); assertTrue(output.contains("Found duplicated address: localhost:8080.")); } @Test - public void testDuplicatePeersId() { + public void testDuplicatePeersId() throws IOException { String[] args = {"local", "raftMetaConf", "-peers", "peer1|localhost:8080,peer1|localhost:8081", "-path", "/path"}; String output = execute(args); assertTrue(output.contains("Found duplicated ID: peer1.")); From a6d205606bdb054597c8ef9fc51f2cb87e8c050e Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Thu, 19 Sep 2024 22:43:53 +0530 Subject: [PATCH 09/10] Fixed minor issues - 2 --- .../hadoop/ozone/ratis/package-info.java | 22 ------------------- .../ozone/{ratis => shell}/OzoneRatis.java | 7 +++--- .../hadoop/ozone/ratis/package-info.java | 22 ------------------- .../{ratis => shell}/TestOzoneRatis.java | 2 +- 4 files changed, 5 insertions(+), 48 deletions(-) delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java rename hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{ratis => shell}/OzoneRatis.java (88%) delete mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java rename hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/{ratis => shell}/TestOzoneRatis.java (99%) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java deleted file mode 100644 index a202f58438e5..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Ozone Ratis tool. - */ -package org.apache.hadoop.ozone.ratis; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/OzoneRatis.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneRatis.java similarity index 88% rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/OzoneRatis.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneRatis.java index 268ee2b0e120..5bc98268064c 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/ratis/OzoneRatis.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneRatis.java @@ -15,11 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.ratis; +package org.apache.hadoop.ozone.shell; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.tracing.TracingUtil; -import org.apache.hadoop.ozone.shell.Shell; import org.apache.ratis.shell.cli.sh.RatisShell; import picocli.CommandLine; @@ -28,7 +27,7 @@ * Ozone Ratis Command line tool. */ @CommandLine.Command(name = "ozone ratis", - description = "Shell for Ozone Ratis Operations", + description = "Shell for running Ratis commands", versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true) public class OzoneRatis extends Shell { @@ -51,6 +50,8 @@ public int execute(String[] argv) { TracingUtil.initTracing("shell", createOzoneConfiguration()); String spanName = "ozone ratis" + String.join(" ", argv); return TracingUtil.executeInNewSpan(spanName, () -> { + // TODO: When Ozone has RATIS-2155, update this line to use the RatisShell.Builder + // in order to setup TLS and other confs. final RatisShell shell = new RatisShell(System.out); return shell.run(argv); }); diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java deleted file mode 100644 index c585018c5f1f..000000000000 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Tests for Ozone Ratis. - */ -package org.apache.hadoop.ozone.ratis; diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneRatis.java similarity index 99% rename from hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneRatis.java index 4d9e3acd3296..9c27bedcf7da 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/ratis/TestOzoneRatis.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneRatis.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.ratis; +package org.apache.hadoop.ozone.shell; import org.apache.ratis.proto.RaftProtos; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; From 6dcecb0ab2f21ae0f6c2487e18410722b3e7f698 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Fri, 20 Sep 2024 09:09:26 +0530 Subject: [PATCH 10/10] Changed the package name --- hadoop-ozone/dist/src/shell/ozone/ozone | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 9e9d70bcb45a..0d005b3bd785 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -233,7 +233,7 @@ function ozonecmd_case OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; ratis) - OZONE_CLASSNAME=org.apache.hadoop.ozone.ratis.OzoneRatis + OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.OzoneRatis OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; *)