From 41b7fcadb9c6fb06d339bb92dcbe1cc001eee19c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Mon, 1 Mar 2021 10:34:30 +0100 Subject: [PATCH 1/5] ec proto changes --- .../src/main/proto/DatanodeClientProtocol.proto | 3 +++ .../interface-client/src/main/proto/hdds.proto | 12 ++++++++++++ .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 4 ++++ .../src/main/proto/ScmServerProtocol.proto | 7 +++++++ 4 files changed, 26 insertions(+) diff --git a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto index 01cc4fff0a29..413c558ee65f 100644 --- a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto +++ b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto @@ -154,6 +154,9 @@ message DatanodeBlockID { required int64 containerID = 1; required int64 localID = 2; optional uint64 blockCommitSequenceId = 3 [default = 0]; + //for EC we record the index of the + //container/block inside the EC replication group. + optional int32 replicationIndex = 4; } message KeyValue { diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index 82504492eb8f..ebf9f523ac96 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -228,13 +228,25 @@ enum ReplicationType { RATIS = 1; STAND_ALONE = 2; CHAINED = 3; + EC = 4; } +//Deprected: we should use replication config implementations everywhere enum ReplicationFactor { ONE = 1; THREE = 3; } +//replicaiton config for Ratis replication. Will replace pure ReplicationType everywhere +message RatisReplicationConfig { + required ReplicationFactor factor = 1; +} + +message ECReplicationConfig { + required int32 data = 1; + required int32 parity = 2; +} + enum ScmOps { allocateBlock = 1; keyBlocksInfoList = 2; diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto index 505f9caa4184..029d210896c6 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto @@ -206,6 +206,10 @@ message ContainerReplicaProto { optional int64 deleteTransactionId = 11; optional uint64 blockCommitSequenceId = 12; optional string originNodeId = 13; + //the index of this replica in the replication group. + //this will be different for each instance in case of EC + //but the same (0) for all instances for standard Ratis + optional int32 replicationIndex = 14; } message CommandStatusReportsProto { diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index 5eac5be45171..1bb2e78c2ba6 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -125,10 +125,17 @@ enum Status { message AllocateScmBlockRequestProto { required uint64 size = 1; required uint32 numBlocks = 2; + + //deprected, use RatisReplicationConfig required ReplicationType type = 3; + required hadoop.hdds.ReplicationFactor factor = 4; required string owner = 5; optional ExcludeListProto excludeList = 7; + + //based on replication type, at least one should be included + optional hadoop.hdds.RatisReplicationConfig ratisReplicationConfig = 8; + optional hadoop.hdds.EcReplicationConfig ecReplicationConfig = 8; } /** From a5ec416f7e8d2f9509d1abc12905c916a391dc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Mon, 1 Mar 2021 12:04:03 +0100 Subject: [PATCH 2/5] fixing typo --- .../interface-server/src/main/proto/ScmServerProtocol.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index 1bb2e78c2ba6..8b52035be369 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -135,7 +135,7 @@ message AllocateScmBlockRequestProto { //based on replication type, at least one should be included optional hadoop.hdds.RatisReplicationConfig ratisReplicationConfig = 8; - optional hadoop.hdds.EcReplicationConfig ecReplicationConfig = 8; + optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 9; } /** From 2fe4f34d300fa393089ddc4f2dffa646346e859a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Mon, 8 Mar 2021 10:48:54 +0100 Subject: [PATCH 3/5] fix typo --- .../interface-server/src/main/proto/ScmServerProtocol.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index 8b52035be369..5a7ed4544404 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -126,7 +126,7 @@ message AllocateScmBlockRequestProto { required uint64 size = 1; required uint32 numBlocks = 2; - //deprected, use RatisReplicationConfig + //deprecated, use RatisReplicationConfig required ReplicationType type = 3; required hadoop.hdds.ReplicationFactor factor = 4; From 2589d41c67ac4e00933992fb50a4cac4e2c277fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Tue, 9 Mar 2021 15:11:56 +0100 Subject: [PATCH 4/5] configuration object for standalone replication --- hadoop-hdds/interface-client/src/main/proto/hdds.proto | 4 ++++ .../interface-server/src/main/proto/ScmServerProtocol.proto | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index ebf9f523ac96..7c7f61cb8e00 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -242,6 +242,10 @@ message RatisReplicationConfig { required ReplicationFactor factor = 1; } +message StandaloneReplicationConfig { + required ReplicationFactor factor = 1; +} + message ECReplicationConfig { required int32 data = 1; required int32 parity = 2; diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index 5a7ed4544404..e4c4f5a3ba64 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -129,13 +129,14 @@ message AllocateScmBlockRequestProto { //deprecated, use RatisReplicationConfig required ReplicationType type = 3; - required hadoop.hdds.ReplicationFactor factor = 4; + optional hadoop.hdds.ReplicationFactor factor = 4; required string owner = 5; optional ExcludeListProto excludeList = 7; - //based on replication type, at least one should be included + //based on replication type, at least one of these should be included optional hadoop.hdds.RatisReplicationConfig ratisReplicationConfig = 8; optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 9; + optional hadoop.hdds.StandaloneReplicationConfig standaloneReplicationConfig = 10; } /** From c0e9a2f780b3ca6037d884dc3cf8b98cd693ed18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 10 Mar 2021 11:15:11 +0100 Subject: [PATCH 5/5] move replication indexes to a separated PR --- .../src/main/proto/DatanodeClientProtocol.proto | 3 --- .../src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto | 4 ---- 2 files changed, 7 deletions(-) diff --git a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto index 413c558ee65f..01cc4fff0a29 100644 --- a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto +++ b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto @@ -154,9 +154,6 @@ message DatanodeBlockID { required int64 containerID = 1; required int64 localID = 2; optional uint64 blockCommitSequenceId = 3 [default = 0]; - //for EC we record the index of the - //container/block inside the EC replication group. - optional int32 replicationIndex = 4; } message KeyValue { diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto index 029d210896c6..505f9caa4184 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto @@ -206,10 +206,6 @@ message ContainerReplicaProto { optional int64 deleteTransactionId = 11; optional uint64 blockCommitSequenceId = 12; optional string originNodeId = 13; - //the index of this replica in the replication group. - //this will be different for each instance in case of EC - //but the same (0) for all instances for standard Ratis - optional int32 replicationIndex = 14; } message CommandStatusReportsProto {