From d934186bee19ef6cfbb918862eef8c6c20cd578d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Thu, 25 Mar 2021 13:43:23 +0100 Subject: [PATCH 1/2] replication config --- .../hdds/client/RatisReplicationConfig.java | 63 +++++++++++++++ .../hadoop/hdds/client/ReplicationConfig.java | 79 +++++++++++++++++++ .../client/StandaloneReplicationConfig.java | 62 +++++++++++++++ .../scm/protocol/TestReplicationConfig.java | 66 ++++++++++++++++ 4 files changed, 270 insertions(+) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java new file mode 100644 index 000000000000..e7e647219449 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.client; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; + +import java.util.Objects; + +/** + * Replication configuration for EC replication. + */ +public class RatisReplicationConfig + implements ReplicationConfig { + + private final ReplicationFactor replicationFactor; + + public RatisReplicationConfig(ReplicationFactor replicationFactor) { + this.replicationFactor = replicationFactor; + } + + @Override + public ReplicationType getReplicationType() { + return ReplicationType.RATIS; + } + + public ReplicationFactor getReplicationFactor() { + return replicationFactor; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RatisReplicationConfig that = (RatisReplicationConfig) o; + return replicationFactor == that.replicationFactor; + } + + @Override + public int hashCode() { + return Objects.hash(replicationFactor); + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java new file mode 100644 index 000000000000..3003ae868ec8 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.client; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + +/** + * Replication configuration for any ReplicationType with all the required + * parameters.. + */ +public interface ReplicationConfig { + + /** + * Helper method to create proper replication method from old-style + * factor+type definition. + *

+ * Note: it's never used for EC replication where config is created. + */ + static ReplicationConfig fromTypeAndFactor( + HddsProtos.ReplicationType type, + HddsProtos.ReplicationFactor factor + ) { + switch (type) { + case RATIS: + return new RatisReplicationConfig(factor); + case STAND_ALONE: + return new StandaloneReplicationConfig(factor); + default: + throw new UnsupportedOperationException( + "Not supported replication: " + type); + } + } + + /** + * Helper method to serialize from proto. + *

+ * This uses either the old type/factor or the new ecConfig depends on the + * type. + *

+ * Note: It will support all the available replication types (including EC). + *

+ * Separated to remain be synced with the EC feature branch, as later it + * will have different signature. + */ + static ReplicationConfig fromProto( + HddsProtos.ReplicationType type, + HddsProtos.ReplicationFactor factor) { + switch (type) { + case RATIS: + case STAND_ALONE: + return fromTypeAndFactor(type, factor); + default: + throw new UnsupportedOperationException( + "Not supported replication: " + type); + } + } + + /** + * Replication type supported by the replication config. + */ + HddsProtos.ReplicationType getReplicationType(); + +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java new file mode 100644 index 000000000000..766ae1c5774f --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.client; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; + +import java.util.Objects; + +/** + * Replication configuration for STANDALONE replication. + */ +public class StandaloneReplicationConfig implements ReplicationConfig { + + private final ReplicationFactor replicationFactor; + + public StandaloneReplicationConfig(ReplicationFactor replicationFactor) { + this.replicationFactor = replicationFactor; + } + + public ReplicationFactor getReplicationFactor() { + return replicationFactor; + } + + @Override + public ReplicationType getReplicationType() { + return ReplicationType.STAND_ALONE; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StandaloneReplicationConfig that = (StandaloneReplicationConfig) o; + return replicationFactor == that.replicationFactor; + } + + @Override + public int hashCode() { + return Objects.hash(replicationFactor); + } +} diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java new file mode 100644 index 000000000000..5c00923e2b66 --- /dev/null +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.protocol; + +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test replicationConfig. + */ +public class TestReplicationConfig { + + @Test + public void deserializeRatis() { + final ReplicationConfig replicationConfig = ReplicationConfig + .fromProto(ReplicationType.RATIS, ReplicationFactor.THREE); + + Assert + .assertEquals(RatisReplicationConfig.class, + replicationConfig.getClass()); + + RatisReplicationConfig ratisReplicationConfig = + (RatisReplicationConfig) replicationConfig; + Assert.assertEquals(ReplicationType.RATIS, + ratisReplicationConfig.getReplicationType()); + Assert.assertEquals(ReplicationFactor.THREE, + ratisReplicationConfig.getReplicationFactor()); + } + + @Test + public void deserializeStandalone() { + final ReplicationConfig replicationConfig = ReplicationConfig + .fromProto(ReplicationType.STAND_ALONE, ReplicationFactor.ONE); + + Assert + .assertEquals(StandaloneReplicationConfig.class, + replicationConfig.getClass()); + + StandaloneReplicationConfig standalone = + (StandaloneReplicationConfig) replicationConfig; + Assert.assertEquals(ReplicationType.STAND_ALONE, + standalone.getReplicationType()); + Assert.assertEquals(ReplicationFactor.ONE, + standalone.getReplicationFactor()); + } +} \ No newline at end of file From 3a406d1759dd226748068cdce53f4ee667e556ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Mon, 29 Mar 2021 10:35:58 +0200 Subject: [PATCH 2/2] add new methods from scm refactor --- .../hdds/client/RatisReplicationConfig.java | 19 +++++++++++++++++++ .../hadoop/hdds/client/ReplicationConfig.java | 19 +++++++++++++++++++ .../client/StandaloneReplicationConfig.java | 10 ++++++++++ .../scm/protocol/TestReplicationConfig.java | 4 ++-- 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java index e7e647219449..8f299059cd56 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java @@ -35,11 +35,25 @@ public RatisReplicationConfig(ReplicationFactor replicationFactor) { this.replicationFactor = replicationFactor; } + public static boolean hasFactor(ReplicationConfig replicationConfig, + ReplicationFactor factor) { + if (replicationConfig instanceof RatisReplicationConfig) { + return ((RatisReplicationConfig) replicationConfig).getReplicationFactor() + .equals(factor); + } + return false; + } + @Override public ReplicationType getReplicationType() { return ReplicationType.RATIS; } + @Override + public int getRequiredNodes() { + return replicationFactor.getNumber(); + } + public ReplicationFactor getReplicationFactor() { return replicationFactor; } @@ -56,6 +70,11 @@ public boolean equals(Object o) { return replicationFactor == that.replicationFactor; } + @Override + public String toString() { + return "RATIS/" + replicationFactor; + } + @Override public int hashCode() { return Objects.hash(replicationFactor); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java index 3003ae868ec8..7818661e25ae 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java @@ -71,9 +71,28 @@ static ReplicationConfig fromProto( } } + static HddsProtos.ReplicationFactor getLegacyFactor( + ReplicationConfig replicationConfig) { + if (replicationConfig instanceof RatisReplicationConfig) { + return ((RatisReplicationConfig) replicationConfig) + .getReplicationFactor(); + } else if (replicationConfig instanceof StandaloneReplicationConfig) { + return ((StandaloneReplicationConfig) replicationConfig) + .getReplicationFactor(); + } + throw new UnsupportedOperationException( + "factor is not valid property of replication " + replicationConfig + .getReplicationType()); + } + /** * Replication type supported by the replication config. */ HddsProtos.ReplicationType getReplicationType(); + /** + * Number of required nodes for this replication. + */ + int getRequiredNodes(); + } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java index 766ae1c5774f..49f8ebd8ebb0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java @@ -38,6 +38,11 @@ public ReplicationFactor getReplicationFactor() { return replicationFactor; } + @Override + public int getRequiredNodes() { + return replicationFactor.getNumber(); + } + @Override public ReplicationType getReplicationType() { return ReplicationType.STAND_ALONE; @@ -55,6 +60,11 @@ public boolean equals(Object o) { return replicationFactor == that.replicationFactor; } + @Override + public String toString() { + return "STANDALONE/" + replicationFactor; + } + @Override public int hashCode() { return Objects.hash(replicationFactor); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java index 5c00923e2b66..4ab60bc9cc8e 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java @@ -33,7 +33,7 @@ public class TestReplicationConfig { @Test public void deserializeRatis() { final ReplicationConfig replicationConfig = ReplicationConfig - .fromProto(ReplicationType.RATIS, ReplicationFactor.THREE); + .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE); Assert .assertEquals(RatisReplicationConfig.class, @@ -50,7 +50,7 @@ public void deserializeRatis() { @Test public void deserializeStandalone() { final ReplicationConfig replicationConfig = ReplicationConfig - .fromProto(ReplicationType.STAND_ALONE, ReplicationFactor.ONE); + .fromTypeAndFactor(ReplicationType.STAND_ALONE, ReplicationFactor.ONE); Assert .assertEquals(StandaloneReplicationConfig.class,