diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java index 83abb937b03e..584d00d691ff 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java @@ -182,7 +182,7 @@ protected BlockExtendedInputStream getOrOpenStream(int locationIndex) throws IOE BlockLocationInfo blkInfo = new BlockLocationInfo.Builder() .setBlockID(blockInfo.getBlockID()) - .setLength(internalBlockLength(locationIndex + 1)) + .setLength(internalBlockLength(locationIndex + 1, this.repConfig, this.blockInfo.getLength())) .setPipeline(blockInfo.getPipeline()) .setToken(blockInfo.getToken()) .setPartNumber(blockInfo.getPartNumber()) @@ -238,11 +238,21 @@ protected Function ecPipelineRefreshFunction( * Returns the length of the Nth block in the block group, taking account of a * potentially partial last stripe. Note that the internal block index is * numbered starting from 1. - * @param index - Index number of the internal block, starting from 1 + * @param index index number of the internal block, starting from 1. + * @param repConfig EC replication config. + * @param length length of the whole block group. */ - protected long internalBlockLength(int index) { - long lastStripe = blockInfo.getLength() % stripeSize; - long blockSize = (blockInfo.getLength() - lastStripe) / repConfig.getData(); + public static long internalBlockLength(int index, ECReplicationConfig repConfig, long length) { + if (index <= 0) { + throw new IllegalArgumentException("Index must start from 1."); + } + if (length < 0) { + throw new IllegalArgumentException("Block length cannot be negative."); + } + long ecChunkSize = (long) repConfig.getEcChunkSize(); + long stripeSize = ecChunkSize * repConfig.getData(); + long lastStripe = length % stripeSize; + long blockSize = (length - lastStripe) / repConfig.getData(); long lastCell = lastStripe / ecChunkSize + 1; long lastCellLength = lastStripe % ecChunkSize; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 3d52e3f6972c..2923ee98cad7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfigValidator; @@ -94,6 +95,7 @@ import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.client.io.ECBlockInputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -1540,6 +1542,17 @@ public OzoneInputStream getKey( return getInputStreamWithRetryFunction(keyInfo); } + /** + * Returns a map that contains {@link org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo} objects of the given key + * as keys of the map. + * Values of the returned map are internal blocks of {@link org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo}. + * The blocks are represented by another map of {@link org.apache.hadoop.hdds.protocol.DatanodeDetails} as keys and + * {@link org.apache.hadoop.ozone.client.io.OzoneInputStream} as values. + * replicaitonConfig in dnKeyInfo is used to instantiate an input stream for each block. + * In case of EC, ECBlockInputStream is instantiated and causes an error later when the stream is processed. + * To prevent such an error, RATIS ONE replication is used instead and the length of each block is calculated by a + * helper method. + */ @Override public Map > getKeysEveryReplicas(String volumeName, @@ -1554,6 +1567,7 @@ public OzoneInputStream getKey( OmKeyInfo keyInfo = getKeyInfo(volumeName, bucketName, keyName, true); List keyLocationInfos = keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly(); + ReplicationConfig replicationConfig = keyInfo.getReplicationConfig(); for (OmKeyLocationInfo locationInfo : keyLocationInfos) { Map blocks = new HashMap<>(); @@ -1568,9 +1582,13 @@ public OzoneInputStream getKey( Pipeline pipeline = new Pipeline.Builder(pipelineBefore).setNodes(nodes) .setId(PipelineID.randomId()).build(); + long length = replicationConfig instanceof ECReplicationConfig + ? ECBlockInputStream.internalBlockLength(pipelineBefore.getReplicaIndex(dn), + (ECReplicationConfig) replicationConfig, locationInfo.getLength()) + : locationInfo.getLength(); OmKeyLocationInfo dnKeyLocation = new OmKeyLocationInfo.Builder() .setBlockID(locationInfo.getBlockID()) - .setLength(locationInfo.getLength()) + .setLength(length) .setOffset(locationInfo.getOffset()) .setToken(locationInfo.getToken()) .setPartNumber(locationInfo.getPartNumber()) @@ -1594,7 +1612,9 @@ public OzoneInputStream getKey( .setDataSize(keyInfo.getDataSize()) .setCreationTime(keyInfo.getCreationTime()) .setModificationTime(keyInfo.getModificationTime()) - .setReplicationConfig(keyInfo.getReplicationConfig()) + .setReplicationConfig(replicationConfig instanceof ECReplicationConfig + ? RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE) + : keyInfo.getReplicationConfig()) .setFileEncryptionInfo(keyInfo.getFileEncryptionInfo()) .setAcls(keyInfo.getAcls()) .setObjectID(keyInfo.getObjectID()) diff --git a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh index 3111177af753..cd129bb07872 100755 --- a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh @@ -46,3 +46,7 @@ execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-dead-datanode. docker start "${container}" wait_for_datanode "${container}" HEALTHY 60 + +start_docker_env 9 +execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot +execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot new file mode 100644 index 000000000000..5b3638040a73 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone Debug CLI for EC(3,2) replicated keys +Library OperatingSystem +Library Process +Resource ../lib/os.robot +Resource ozone-debug.robot +Test Timeout 5 minute +Suite Setup Create Volume Bucket + +*** Variables *** +${PREFIX} ${EMPTY} +${VOLUME} cli-debug-volume${PREFIX} +${BUCKET} cli-debug-bucket +${TESTFILE} testfile +${EC_DATA} 3 +${EC_PARITY} 2 + +*** Keywords *** +Create Volume Bucket + Execute ozone sh volume create o3://om/${VOLUME} + Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET} + +Create EC key + [arguments] ${bs} ${count} + + Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count} + Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC + +*** Test Cases *** +0 data block + Create EC key 1000 0 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 1 + +1 data block + Create EC key 1048576 1 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 6 + ${sum_size} = Evaluate 1048576 * 3 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +2 data blocks + Create EC key 1048576 2 + ${directory} = Execute read-replicas CLI tool + ${sum_size} = Evaluate 1048576 * 4 + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 6 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +3 data blocks + Create EC key 1048576 3 + ${directory} = Execute read-replicas CLI tool + ${sum_size} = Evaluate 1048576 * 5 + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 6 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +3 data blocks and partial stripe + Create EC key 1000000 4 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 5 + ${sum_size_last_stripe} = Evaluate ((1000000 * 4) % 1048576) * 3 + Should Be Equal As Integers ${count_files} 11 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + +4 data blocks and partial stripe + Create EC key 1000000 5 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 5 + ${sum_size_last_stripe} = Evaluate 1048576 * 3 + ((1000000 * 5) % 1048576) + Should Be Equal As Integers ${count_files} 11 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + +6 data blocks + Create EC key 1048576 6 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 5 + Should Be Equal As Integers ${count_files} 11 + FOR ${block} IN RANGE 1 3 + Verify Healthy EC Replica ${directory} ${block} ${sum_size} + END diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot new file mode 100644 index 000000000000..692f2791e20f --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone Debug CLI for EC(6,3) replicated key +Library OperatingSystem +Library Process +Resource ../lib/os.robot +Resource ozone-debug.robot +Test Timeout 5 minute + +*** Variables *** +${PREFIX} ${EMPTY} +${VOLUME} cli-debug-volume${PREFIX} +${BUCKET} cli-debug-bucket +${TESTFILE} testfile +${EC_DATA} 6 +${EC_PARITY} 3 + +*** Keywords *** +Create EC key + [arguments] ${bs} ${count} + + Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count} + Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC + +*** Test Cases *** +0 data block + Create EC key 1048576 0 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 1 + +1 data block + Create EC key 1048576 1 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 10 + ${sum_size} = Evaluate 1048576 * 4 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +2 data blocks + Create EC key 1048576 2 + ${directory} = Execute read-replicas CLI tool + ${sum_size} = Evaluate 1048576 * 5 + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +3 data blocks + Create EC key 1048576 3 + ${directory} = Execute read-replicas CLI tool + ${sum_size} = Evaluate 1048576 * 6 + ${count_files} = Count Files In Directory ${directory} + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +4 data blocks + Create EC key 1048576 4 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 7 + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +5 data blocks + Create EC key 1048576 5 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 8 + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +6 data blocks + Create EC key 1048576 6 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 9 + Should Be Equal As Integers ${count_files} 10 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + +6 data blocks and partial stripe + Create EC key 1000000 7 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 9 + ${sum_size_last_stripe} = Evaluate ((1000000 * 7) % 1048576) * 4 + Should Be Equal As Integers ${count_files} 19 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + +7 data blocks and partial stripe + Create EC key 1000000 8 + ${directory} = Execute read-replicas CLI tool + ${count_files} = Count Files In Directory ${directory} + ${sum_size} = Evaluate 1048576 * 9 + ${sum_size_last_stripe} = Evaluate 1048576 * 4 + ((1000000 * 8) % 1048576) + Should Be Equal As Integers ${count_files} 19 + Verify Healthy EC Replica ${directory} 1 ${sum_size} + Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot index e0964e4c160b..fb3e0f415860 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot @@ -70,6 +70,12 @@ Verify Healthy Replica ${md5sum} = Execute cat ${block_filenames} | md5sum | awk '{print $1}' Should Be Equal ${md5sum} ${expected_md5sum} +Verify Healthy EC Replica + [arguments] ${directory} ${block} ${expected_block_size} + + ${block_size} = Execute ls -l ${directory} | grep "testfile_block${block}_ozone-datanode-.*\.ozone_default" | awk '{sum += $5} END {print sum}' + Should Be Equal As Integers ${block_size} ${expected_block_size} + Verify Corrupt Replica [arguments] ${json} ${replica} ${valid_md5sum}