From bc8dc6bb30af984620842e8895b61e91546d422d Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Fri, 18 Oct 2024 16:55:41 +0200 Subject: [PATCH 1/9] add if clause in case of EC replicas in RpcClent.java --- .../hadoop/ozone/client/rpc/RpcClient.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 8445f9e6954c..851951123edb 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -34,12 +34,7 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.Syncable; -import org.apache.hadoop.hdds.client.DefaultReplicationConfig; -import org.apache.hadoop.hdds.client.ECReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfigValidator; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.client.*; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.StorageType; @@ -1598,7 +1593,6 @@ public OzoneInputStream getKey( Pipeline pipelineBefore = locationInfo.getPipeline(); List datanodes = pipelineBefore.getNodes(); - for (DatanodeDetails dn : datanodes) { List nodes = new ArrayList<>(); nodes.add(dn); @@ -1627,7 +1621,7 @@ public OzoneInputStream getKey( .setVolumeName(keyInfo.getVolumeName()) .setBucketName(keyInfo.getBucketName()) .setKeyName(keyInfo.getKeyName()) - .setOmKeyLocationInfos(keyInfo.getKeyLocationVersions()) + .setOmKeyLocationInfos(keyInfo.getKeyLocationVersions()) // configure the getKeyLocationVersions with a proper length of a block size .setDataSize(keyInfo.getDataSize()) .setCreationTime(keyInfo.getCreationTime()) .setModificationTime(keyInfo.getModificationTime()) @@ -1640,9 +1634,13 @@ public OzoneInputStream getKey( .setFileChecksum(keyInfo.getFileChecksum()) .setOwnerName(keyInfo.getOwnerName()) .build(); + dnKeyInfo.setMetadata(keyInfo.getMetadata()); dnKeyInfo.setKeyLocationVersions(keyLocationInfoGroups); - + if (dnKeyInfo.getReplicationConfig() instanceof ECReplicationConfig) { + dnKeyInfo.setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); + } + // blocks.put(dn, createInputStream(dnKeyInfo, Function.identity())); } From 1104f7c2dbe88465fa2f1890cccb7d233ca9c766 Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Wed, 4 Dec 2024 02:34:03 +0100 Subject: [PATCH 2/9] Tweaked read-replicas to handle EC replicated keys and added robot tests --- .../hadoop/ozone/client/rpc/RpcClient.java | 41 +++++-- hadoop-ozone/dist/src/main/compose/testlib.sh | 3 + .../debug/ozone-debug-tests-ec3-2.robot | 103 ++++++++++++++++ .../debug/ozone-debug-tests-ec6-3.robot | 112 ++++++++++++++++++ .../main/smoketest/debug/ozone-debug.robot | 8 +- 5 files changed, 258 insertions(+), 9 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 851951123edb..3eb4cfe5a940 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1586,7 +1586,7 @@ public OzoneInputStream getKey( OmKeyInfo keyInfo = getKeyInfo(volumeName, bucketName, keyName, true); List keyLocationInfos = keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly(); - + ReplicationConfig replicationConfig = keyInfo.getReplicationConfig(); for (OmKeyLocationInfo locationInfo : keyLocationInfos) { Map blocks = new HashMap<>(); @@ -1599,9 +1599,12 @@ public OzoneInputStream getKey( Pipeline pipeline = new Pipeline.Builder(pipelineBefore).setNodes(nodes) .setId(PipelineID.randomId()).build(); + long length = replicationConfig instanceof ECReplicationConfig + ? internalBlockLength(pipelineBefore.getReplicaIndex(dn), (ECReplicationConfig) replicationConfig, locationInfo.getLength()) + : locationInfo.getLength(); OmKeyLocationInfo dnKeyLocation = new OmKeyLocationInfo.Builder() .setBlockID(locationInfo.getBlockID()) - .setLength(locationInfo.getLength()) + .setLength(length) .setOffset(locationInfo.getOffset()) .setToken(locationInfo.getToken()) .setPartNumber(locationInfo.getPartNumber()) @@ -1621,11 +1624,13 @@ public OzoneInputStream getKey( .setVolumeName(keyInfo.getVolumeName()) .setBucketName(keyInfo.getBucketName()) .setKeyName(keyInfo.getKeyName()) - .setOmKeyLocationInfos(keyInfo.getKeyLocationVersions()) // configure the getKeyLocationVersions with a proper length of a block size + .setOmKeyLocationInfos(keyInfo.getKeyLocationVersions()) .setDataSize(keyInfo.getDataSize()) .setCreationTime(keyInfo.getCreationTime()) .setModificationTime(keyInfo.getModificationTime()) - .setReplicationConfig(keyInfo.getReplicationConfig()) + .setReplicationConfig(replicationConfig instanceof ECReplicationConfig + ? RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE) + : keyInfo.getReplicationConfig()) // We don't want to create ECBlockInputStream when calling createInputStream .setFileEncryptionInfo(keyInfo.getFileEncryptionInfo()) .setAcls(keyInfo.getAcls()) .setObjectID(keyInfo.getObjectID()) @@ -1637,10 +1642,6 @@ public OzoneInputStream getKey( dnKeyInfo.setMetadata(keyInfo.getMetadata()); dnKeyInfo.setKeyLocationVersions(keyLocationInfoGroups); - if (dnKeyInfo.getReplicationConfig() instanceof ECReplicationConfig) { - dnKeyInfo.setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); - } - // blocks.put(dn, createInputStream(dnKeyInfo, Function.identity())); } @@ -1650,6 +1651,30 @@ public OzoneInputStream getKey( return result; } + // Helper method to calculate the internal block length of an EC-replicated key. + private long internalBlockLength(int index, ECReplicationConfig ecRepConfig, long blockLength) { + long ecChunkSize = (long) ecRepConfig.getEcChunkSize(); + long stripeSize = ecChunkSize * ecRepConfig.getData(); + long lastStripe = blockLength % stripeSize; + long blockSize = (blockLength - lastStripe) / ecRepConfig.getData(); + long lastCell = lastStripe / ecChunkSize + 1; + long lastCellLength = lastStripe % ecChunkSize; + if (index > ecRepConfig.getData()) { + // Parity blocks and their size are driven by the size of the + // first block of the block group. All parity blocks have the same size + // as block_1. + index = 1; + } + + if (index < lastCell) { + return blockSize + ecChunkSize; + } else if (index == lastCell) { + return blockSize + lastCellLength; + } else { + return blockSize; + } + } + @Override public void deleteKey( String volumeName, String bucketName, String keyName, boolean recursive) diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index 1e9cc85781a6..586a685eb08b 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -575,6 +575,9 @@ execute_debug_tests() { docker start "${container}" wait_for_datanode "${container}" HEALTHY 60 + start_docker_env 9 + execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot + execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot } ## @description Wait for datanode state diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot new file mode 100644 index 000000000000..5b3638040a73 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone Debug CLI for EC(3,2) replicated keys +Library OperatingSystem +Library Process +Resource ../lib/os.robot +Resource ozone-debug.robot +Test Timeout 5 minute +Suite Setup Create Volume Bucket + +*** Variables *** +${PREFIX} ${EMPTY} +${VOLUME} cli-debug-volume${PREFIX} +${BUCKET} cli-debug-bucket +${TESTFILE} testfile +${EC_DATA} 3 +${EC_PARITY} 2 + +*** Keywords *** +Create Volume Bucket + Execute ozone sh volume create o3://om/${VOLUME} + Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET} + +Create EC key + [arguments] ${bs} ${count} + + Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count} + Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC + +*** Test Cases *** +0 data block + Create EC key 1000 0 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 1 + +1 data block + Create EC key 1048576 1 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 6 + ${sum_size} = Evaluate 1048576 * 3 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +2 data blocks + Create EC key 1048576 2 + ${directory} = Execute read-replicas CLI tool + ${sum_size} = Evaluate 1048576 * 4 + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 6 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +3 data blocks + Create EC key 1048576 3 + ${directory} = Execute read-replicas CLI tool + ${sum_size} = Evaluate 1048576 * 5 + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 6 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +3 data blocks and partial stripe + Create EC key 1000000 4 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 5 + ${sum_size_last_stripe} = Evaluate ((1000000 * 4) % 1048576) * 3 + Should Be Equal As Integers ${count_files} 11 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + +4 data blocks and partial stripe + Create EC key 1000000 5 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 5 + ${sum_size_last_stripe} = Evaluate 1048576 * 3 + ((1000000 * 5) % 1048576) + Should Be Equal As Integers ${count_files} 11 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + +6 data blocks + Create EC key 1048576 6 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 5 + Should Be Equal As Integers ${count_files} 11 + FOR ${block} IN RANGE 1 3 + Verify Healthy EC Replica ${directory} ${block} ${sum_size} + END diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot new file mode 100644 index 000000000000..692f2791e20f --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone Debug CLI for EC(6,3) replicated key +Library OperatingSystem +Library Process +Resource ../lib/os.robot +Resource ozone-debug.robot +Test Timeout 5 minute + +*** Variables *** +${PREFIX} ${EMPTY} +${VOLUME} cli-debug-volume${PREFIX} +${BUCKET} cli-debug-bucket +${TESTFILE} testfile +${EC_DATA} 6 +${EC_PARITY} 3 + +*** Keywords *** +Create EC key + [arguments] ${bs} ${count} + + Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count} + Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC + +*** Test Cases *** +0 data block + Create EC key 1048576 0 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 1 + +1 data block + Create EC key 1048576 1 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 10 + ${sum_size} = Evaluate 1048576 * 4 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +2 data blocks + Create EC key 1048576 2 + ${directory} = Execute read-replicas CLI tool + ${sum_size} = Evaluate 1048576 * 5 + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +3 data blocks + Create EC key 1048576 3 + ${directory} = Execute read-replicas CLI tool + ${sum_size} = Evaluate 1048576 * 6 + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +4 data blocks + Create EC key 1048576 4 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 7 + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +5 data blocks + Create EC key 1048576 5 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 8 + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +6 data blocks + Create EC key 1048576 6 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 9 + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +6 data blocks and partial stripe + Create EC key 1000000 7 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 9 + ${sum_size_last_stripe} = Evaluate ((1000000 * 7) % 1048576) * 4 + Should Be Equal As Integers ${count_files} 19 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + +7 data blocks and partial stripe + Create EC key 1000000 8 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 9 + ${sum_size_last_stripe} = Evaluate 1048576 * 4 + ((1000000 * 8) % 1048576) + Should Be Equal As Integers ${count_files} 19 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot index fa915819eea7..aeb0e6459db5 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot @@ -65,7 +65,7 @@ Get Replica Filenames END ${filenames} = Catenate @{list} - + [return] ${filenames} Verify Healthy Replica @@ -75,6 +75,12 @@ Verify Healthy Replica ${md5sum} = Execute cat ${block_filenames} | md5sum | awk '{print $1}' Should Be Equal ${md5sum} ${expected_md5sum} +Verify Healthy EC Replica + [arguments] ${directory} ${block} ${expected_block_size} + + ${block_size} = Execute ls -l ${directory} | grep "testfile_block${block}_ozone-datanode-.*\.ozone_default" | awk '{sum += $5} END {print sum}' + Should Be Equal As Integers ${block_size} ${expected_block_size} + Verify Corrupt Replica [arguments] ${json} ${replica} ${valid_md5sum} From e873c4d92d040a20afb4733a7651965508c5a361 Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Wed, 4 Dec 2024 03:11:38 +0100 Subject: [PATCH 3/9] avoided using '*' form of import --- .../org/apache/hadoop/ozone/client/rpc/RpcClient.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 3eb4cfe5a940..a11932fa1423 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -34,7 +34,13 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.Syncable; -import org.apache.hadoop.hdds.client.*; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfigValidator; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.StorageType; From 24a10b18406af0c3c26d2b966c393f1e193cfd18 Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Wed, 4 Dec 2024 09:57:25 +0100 Subject: [PATCH 4/9] fixed checkstyle --- .../java/org/apache/hadoop/ozone/client/rpc/RpcClient.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index a11932fa1423..48ebaa5e301e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1606,7 +1606,8 @@ public OzoneInputStream getKey( = new Pipeline.Builder(pipelineBefore).setNodes(nodes) .setId(PipelineID.randomId()).build(); long length = replicationConfig instanceof ECReplicationConfig - ? internalBlockLength(pipelineBefore.getReplicaIndex(dn), (ECReplicationConfig) replicationConfig, locationInfo.getLength()) + ? internalBlockLength(pipelineBefore.getReplicaIndex(dn), + (ECReplicationConfig) replicationConfig, locationInfo.getLength()) : locationInfo.getLength(); OmKeyLocationInfo dnKeyLocation = new OmKeyLocationInfo.Builder() .setBlockID(locationInfo.getBlockID()) @@ -1636,7 +1637,7 @@ public OzoneInputStream getKey( .setModificationTime(keyInfo.getModificationTime()) .setReplicationConfig(replicationConfig instanceof ECReplicationConfig ? RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE) - : keyInfo.getReplicationConfig()) // We don't want to create ECBlockInputStream when calling createInputStream + : keyInfo.getReplicationConfig()) .setFileEncryptionInfo(keyInfo.getFileEncryptionInfo()) .setAcls(keyInfo.getAcls()) .setObjectID(keyInfo.getObjectID()) From 37d4d74f059da71a1384428110f3a4983d7057a0 Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Sun, 15 Dec 2024 17:46:13 +0100 Subject: [PATCH 5/9] Undo unecessary removal and insertion of lines, and add javadocs --- .../hadoop/ozone/client/rpc/RpcClient.java | 20 ++++++++++++++++++- .../main/smoketest/debug/ozone-debug.robot | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 48ebaa5e301e..342175fc8f36 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1578,6 +1578,17 @@ public OzoneInputStream getKey( return getInputStreamWithRetryFunction(keyInfo); } + /** + * Returns a map that contains {@link org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo} objects of the given key + * as keys of the map. + * Values of the returned map are internal blocks of {@link org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo}. + * The blocks are represented by another map of {@link org.apache.hadoop.hdds.protocol.DatanodeDetails} as keys and + * {@link org.apache.hadoop.ozone.client.io.OzoneInputStream} as values. + * replicaitonConfig in dnKeyInfo is used to instantiate an input stream for each block. + * In case of EC, ECBlockInputStream is instantiated and causes an error later when the stream is processed. + * To prevent such an error, RATIS ONE replication is used instead and the length of each block is calculated by a + * helper method. + */ @Override public Map > getKeysEveryReplicas(String volumeName, @@ -1593,12 +1604,14 @@ public OzoneInputStream getKey( List keyLocationInfos = keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly(); ReplicationConfig replicationConfig = keyInfo.getReplicationConfig(); + for (OmKeyLocationInfo locationInfo : keyLocationInfos) { Map blocks = new HashMap<>(); Pipeline pipelineBefore = locationInfo.getPipeline(); List datanodes = pipelineBefore.getNodes(); + for (DatanodeDetails dn : datanodes) { List nodes = new ArrayList<>(); nodes.add(dn); @@ -1649,6 +1662,7 @@ public OzoneInputStream getKey( dnKeyInfo.setMetadata(keyInfo.getMetadata()); dnKeyInfo.setKeyLocationVersions(keyLocationInfoGroups); + blocks.put(dn, createInputStream(dnKeyInfo, Function.identity())); } @@ -1658,7 +1672,11 @@ public OzoneInputStream getKey( return result; } - // Helper method to calculate the internal block length of an EC-replicated key. + /** + * Helper method to calculate the internal block length of an EC-replicated key. + * The length of data blocks differs depending on whether it is the last strip or not. + * That of parity blocks can be determined by the data blocks. + */ private long internalBlockLength(int index, ECReplicationConfig ecRepConfig, long blockLength) { long ecChunkSize = (long) ecRepConfig.getEcChunkSize(); long stripeSize = ecChunkSize * ecRepConfig.getData(); diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot index aeb0e6459db5..43a3b8b3321c 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot @@ -65,7 +65,7 @@ Get Replica Filenames END ${filenames} = Catenate @{list} - + [return] ${filenames} Verify Healthy Replica From d8b361eda159d6751e623f4dc5ca5b20ad10276e Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Mon, 16 Dec 2024 11:11:27 +0100 Subject: [PATCH 6/9] remove unnecessary line --- .../main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 483e83a21db5..4caebec21b1f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1622,7 +1622,6 @@ public OzoneInputStream getKey( .setFileChecksum(keyInfo.getFileChecksum()) .setOwnerName(keyInfo.getOwnerName()) .build(); - dnKeyInfo.setMetadata(keyInfo.getMetadata()); dnKeyInfo.setKeyLocationVersions(keyLocationInfoGroups); From 512ee5b9184b6471c66f9012388f7a61e70b1a9d Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Wed, 29 Jan 2025 00:26:40 +0100 Subject: [PATCH 7/9] Add EC replica tests to replicas-test.sh --- hadoop-ozone/dist/src/main/compose/common/replicas-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh index 3111177af753..cd129bb07872 100755 --- a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh @@ -46,3 +46,7 @@ execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-dead-datanode. docker start "${container}" wait_for_datanode "${container}" HEALTHY 60 + +start_docker_env 9 +execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot +execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot \ No newline at end of file From 48f044356891c2a0d6661ada10897403251243fb Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Thu, 30 Jan 2025 10:00:10 +0100 Subject: [PATCH 8/9] Make internalBlockLength method public. --- .../ozone/client/io/ECBlockInputStream.java | 16 ++++--- .../hadoop/ozone/client/rpc/RpcClient.java | 43 +------------------ 2 files changed, 13 insertions(+), 46 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java index 83abb937b03e..f24908f82071 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java @@ -182,7 +182,7 @@ protected BlockExtendedInputStream getOrOpenStream(int locationIndex) throws IOE BlockLocationInfo blkInfo = new BlockLocationInfo.Builder() .setBlockID(blockInfo.getBlockID()) - .setLength(internalBlockLength(locationIndex + 1)) + .setLength(internalBlockLength(locationIndex + 1, this.repConfig ,this.blockInfo.getLength())) .setPipeline(blockInfo.getPipeline()) .setToken(blockInfo.getToken()) .setPartNumber(blockInfo.getPartNumber()) @@ -238,11 +238,17 @@ protected Function ecPipelineRefreshFunction( * Returns the length of the Nth block in the block group, taking account of a * potentially partial last stripe. Note that the internal block index is * numbered starting from 1. - * @param index - Index number of the internal block, starting from 1 + * @param index index number of the internal block, starting from 1. + * @param repConfig EC replication config. + * @param length length of the whole block group. */ - protected long internalBlockLength(int index) { - long lastStripe = blockInfo.getLength() % stripeSize; - long blockSize = (blockInfo.getLength() - lastStripe) / repConfig.getData(); + static public long internalBlockLength(int index, ECReplicationConfig repConfig, long length) { + if (index <= 0) throw new IllegalArgumentException("Index must start from 1."); + if (length < 0) throw new IllegalArgumentException("Block length cannot be negative."); + long ecChunkSize = (long) repConfig.getEcChunkSize(); + long stripeSize = ecChunkSize * repConfig.getData(); + long lastStripe = length % stripeSize; + long blockSize = (length - lastStripe) / repConfig.getData(); long lastCell = lastStripe / ecChunkSize + 1; long lastCellLength = lastStripe % ecChunkSize; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index f2dfdba2a18d..42ccd34351be 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -34,6 +34,7 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.Syncable; +import org.apache.hadoop.ozone.client.io.*; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; @@ -83,18 +84,6 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; -import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory; -import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl; -import org.apache.hadoop.ozone.client.io.CipherOutputStreamOzone; -import org.apache.hadoop.ozone.client.io.ECKeyOutputStream; -import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput; -import org.apache.hadoop.ozone.client.io.KeyInputStream; -import org.apache.hadoop.ozone.client.io.KeyOutputStream; -import org.apache.hadoop.ozone.client.io.LengthInputStream; -import org.apache.hadoop.ozone.client.io.OzoneCryptoInputStream; -import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -1582,7 +1571,7 @@ public OzoneInputStream getKey( = new Pipeline.Builder(pipelineBefore).setNodes(nodes) .setId(PipelineID.randomId()).build(); long length = replicationConfig instanceof ECReplicationConfig - ? internalBlockLength(pipelineBefore.getReplicaIndex(dn), + ? ECBlockInputStream.internalBlockLength(pipelineBefore.getReplicaIndex(dn), (ECReplicationConfig) replicationConfig, locationInfo.getLength()) : locationInfo.getLength(); OmKeyLocationInfo dnKeyLocation = new OmKeyLocationInfo.Builder() @@ -1634,34 +1623,6 @@ public OzoneInputStream getKey( return result; } - /** - * Helper method to calculate the internal block length of an EC-replicated key. - * The length of data blocks differs depending on whether it is the last strip or not. - * That of parity blocks can be determined by the data blocks. - */ - private long internalBlockLength(int index, ECReplicationConfig ecRepConfig, long blockLength) { - long ecChunkSize = (long) ecRepConfig.getEcChunkSize(); - long stripeSize = ecChunkSize * ecRepConfig.getData(); - long lastStripe = blockLength % stripeSize; - long blockSize = (blockLength - lastStripe) / ecRepConfig.getData(); - long lastCell = lastStripe / ecChunkSize + 1; - long lastCellLength = lastStripe % ecChunkSize; - if (index > ecRepConfig.getData()) { - // Parity blocks and their size are driven by the size of the - // first block of the block group. All parity blocks have the same size - // as block_1. - index = 1; - } - - if (index < lastCell) { - return blockSize + ecChunkSize; - } else if (index == lastCell) { - return blockSize + lastCellLength; - } else { - return blockSize; - } - } - @Override public void deleteKey( String volumeName, String bucketName, String keyName, boolean recursive) From c2fefa32de93d139f8b5002d6c17fcaab70340a1 Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Thu, 30 Jan 2025 11:41:35 +0100 Subject: [PATCH 9/9] Fixed checkstyle. --- .../hadoop/ozone/client/io/ECBlockInputStream.java | 12 ++++++++---- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 14 +++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java index f24908f82071..584d00d691ff 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java @@ -182,7 +182,7 @@ protected BlockExtendedInputStream getOrOpenStream(int locationIndex) throws IOE BlockLocationInfo blkInfo = new BlockLocationInfo.Builder() .setBlockID(blockInfo.getBlockID()) - .setLength(internalBlockLength(locationIndex + 1, this.repConfig ,this.blockInfo.getLength())) + .setLength(internalBlockLength(locationIndex + 1, this.repConfig, this.blockInfo.getLength())) .setPipeline(blockInfo.getPipeline()) .setToken(blockInfo.getToken()) .setPartNumber(blockInfo.getPartNumber()) @@ -242,9 +242,13 @@ protected Function ecPipelineRefreshFunction( * @param repConfig EC replication config. * @param length length of the whole block group. */ - static public long internalBlockLength(int index, ECReplicationConfig repConfig, long length) { - if (index <= 0) throw new IllegalArgumentException("Index must start from 1."); - if (length < 0) throw new IllegalArgumentException("Block length cannot be negative."); + public static long internalBlockLength(int index, ECReplicationConfig repConfig, long length) { + if (index <= 0) { + throw new IllegalArgumentException("Index must start from 1."); + } + if (length < 0) { + throw new IllegalArgumentException("Block length cannot be negative."); + } long ecChunkSize = (long) repConfig.getEcChunkSize(); long stripeSize = ecChunkSize * repConfig.getData(); long lastStripe = length % stripeSize; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 42ccd34351be..2923ee98cad7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -34,7 +34,6 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.Syncable; -import org.apache.hadoop.ozone.client.io.*; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; @@ -84,6 +83,19 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; +import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory; +import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl; +import org.apache.hadoop.ozone.client.io.CipherOutputStreamOzone; +import org.apache.hadoop.ozone.client.io.ECKeyOutputStream; +import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput; +import org.apache.hadoop.ozone.client.io.KeyInputStream; +import org.apache.hadoop.ozone.client.io.KeyOutputStream; +import org.apache.hadoop.ozone.client.io.LengthInputStream; +import org.apache.hadoop.ozone.client.io.OzoneCryptoInputStream; +import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.client.io.ECBlockInputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException;