diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index d1dcc654b100..0b7c8652d7e7 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java @@ -230,7 +230,7 @@ public enum ChecksumCombineMode { "list rather than full chunk list to optimize performance. " + "Critical to HBase.", tags = ConfigTag.CLIENT) - private boolean incrementalChunkList = false; + private boolean incrementalChunkList = true; @PostConstruct private void validate() { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 42a74dd12c2e..c0a48efead04 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -146,8 +146,8 @@ public final class ScmConfigKeys { "32KB"; public static final String OZONE_CHUNK_LIST_INCREMENTAL = - "ozone.chunk.list.incremental"; - public static final boolean OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT = false; + "ozone.incremental.chunk.list"; + public static final boolean OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT = true; public static final String OZONE_SCM_CONTAINER_LAYOUT_KEY = "ozone.scm.container.layout"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index a0d4b59db168..555b180625c6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -126,7 +126,7 @@ public final class OzoneConfigKeys { public static final String OZONE_FS_HSYNC_ENABLED = "ozone.fs.hsync.enabled"; public static final boolean OZONE_FS_HSYNC_ENABLED_DEFAULT - = false; + = true; /** * hsync lease soft limit. diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 47067de5fede..d9a39d377269 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -856,8 +856,8 @@ - ozone.chunk.list.incremental - false + ozone.incremental.chunk.list + true OZONE, CLIENT, DATANODE, PERFORMANCE By default, a writer client sends full chunk list of a block when it @@ -4116,7 +4116,7 @@ ozone.fs.hsync.enabled - false + true OZONE, CLIENT Enable hsync/hflush. By default they are disabled. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java index e40434f508e6..4690565b0bb1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java @@ -42,7 +42,6 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.BCSID_MISMATCH; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNKNOWN_BCSID; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNSUPPORTED_REQUEST; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_CHUNK_LIST_INCREMENTAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT; @@ -66,6 +65,7 @@ public class BlockManagerImpl implements BlockManager { // Default Read Buffer capacity when Checksum is not present private final int defaultReadBufferCapacity; private final int readMappedBufferThreshold; + private boolean incrementalEnabled; /** * Constructs a Block Manager. @@ -81,6 +81,15 @@ public BlockManagerImpl(ConfigurationSource conf) { this.readMappedBufferThreshold = config.getBufferSize( ScmConfigKeys.OZONE_CHUNK_READ_MAPPED_BUFFER_THRESHOLD_KEY, ScmConfigKeys.OZONE_CHUNK_READ_MAPPED_BUFFER_THRESHOLD_DEFAULT); + incrementalEnabled = + config.getBoolean(OZONE_CHUNK_LIST_INCREMENTAL, + OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT); + if (incrementalEnabled && !VersionedDatanodeFeatures.isFinalized( + HDDSLayoutFeature.HBASE_SUPPORT)) { + LOG.warn("DataNode has not finalized upgrading to a version that " + + "supports incremental chunk list. Fallback to full chunk list"); + incrementalEnabled = false; + } } @Override @@ -93,23 +102,12 @@ public long putBlock(Container container, BlockData data, boolean endOfBlock) throws IOException { return persistPutBlock( (KeyValueContainer) container, - data, - config, - endOfBlock); + data, endOfBlock); } - public static long persistPutBlock(KeyValueContainer container, - BlockData data, ConfigurationSource config, boolean endOfBlock) + public long persistPutBlock(KeyValueContainer container, + BlockData data, boolean endOfBlock) throws IOException { - boolean incrementalEnabled = - config.getBoolean(OZONE_CHUNK_LIST_INCREMENTAL, - OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT); - if (incrementalEnabled && !VersionedDatanodeFeatures.isFinalized( - HDDSLayoutFeature.HBASE_SUPPORT)) { - throw new StorageContainerException("DataNode has not finalized " + - "upgrading to a version that supports incremental chunk list.", - UNSUPPORTED_REQUEST); - } Preconditions.checkNotNull(data, "BlockData cannot be null for put " + "operation."); Preconditions.checkState(data.getContainerID() >= 0, "Container Id " + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java index 3a43ddd8ab09..dbca12c8b26d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java @@ -60,7 +60,6 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl; import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerFactory; -import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager; import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager; import com.codahale.metrics.Timer; @@ -111,6 +110,7 @@ public class GeneratorDatanode extends BaseGenerator { private int overlap; private ChunkManager chunkManager; + private BlockManagerImpl blockManager; private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; @@ -133,7 +133,7 @@ public Void call() throws Exception { config = createOzoneConfiguration(); - BlockManager blockManager = new BlockManagerImpl(config); + blockManager = new BlockManagerImpl(config); chunkManager = ChunkManagerFactory .createChunkManager(config, blockManager, null); @@ -286,7 +286,7 @@ public void generateData(long index) throws Exception { writtenBytes += currentChunkSize; } - BlockManagerImpl.persistPutBlock(container, blockData, config, true); + blockManager.persistPutBlock(container, blockData, true); }