From 74be0459b7fa1ce3b516b163ad4e9925d910e232 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 2 Sep 2016 18:44:30 +0300 Subject: [PATCH] Unmap files to add eagerly in FileSmoother (BACKEND-312). The exact purpose of this is to enable running Druid's IndexMergeBenchmark in Windows, but this is also universally healthy becauseit releases memory determenistically, otherwise files are unmapped when MappedByteBuffers are garbage-collected --- .../com/metamx/common/io/smoosh/FileSmoosher.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/metamx/common/io/smoosh/FileSmoosher.java b/src/main/java/com/metamx/common/io/smoosh/FileSmoosher.java index d741bbb3..776c3a1f 100644 --- a/src/main/java/com/metamx/common/io/smoosh/FileSmoosher.java +++ b/src/main/java/com/metamx/common/io/smoosh/FileSmoosher.java @@ -25,6 +25,7 @@ import com.google.common.io.Closeables; import com.google.common.io.Files; import com.google.common.primitives.Ints; +import com.metamx.common.ByteBufferUtils; import com.metamx.common.IAE; import com.metamx.common.ISE; @@ -40,6 +41,7 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; import java.util.Arrays; @@ -53,6 +55,7 @@ *

* It does not split input files among separate output files, instead the various "chunk" files will * be varying sizes and it is not possible to add a file of size greater than Integer.MAX_VALUE + * TODO(leventov): consider making this class final */ public class FileSmoosher implements Closeable { @@ -105,12 +108,17 @@ public Set getInternalFilenames() public void add(File fileToAdd) throws IOException { - add(fileToAdd.getName(), Files.map(fileToAdd)); + add(fileToAdd.getName(), fileToAdd); } public void add(String name, File fileToAdd) throws IOException { - add(name, Files.map(fileToAdd)); + MappedByteBuffer mappedFileBuffer = Files.map(fileToAdd); + try { + add(name, mappedFileBuffer); + } finally { + ByteBufferUtils.unmap(mappedFileBuffer); + } } public void add(String name, ByteBuffer bufferToAdd) throws IOException