Skip to content

[Backport] Fix resource leaks (ComplexColumn and GenericColumn) (#3629) #3649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fjy merged 1 commit into from
Nov 3, 2016
Merged

[Backport] Fix resource leaks (ComplexColumn and GenericColumn) (#3629) #3649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 26 additions & 25 deletions processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,35 +306,36 @@ private ColumnAnalysis analyzeComplexColumn(
final String typeName
)
{
final ComplexColumn complexColumn = column != null ? column.getComplexColumn() : null;
final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues();
long size = 0;
try (final ComplexColumn complexColumn = column != null ? column.getComplexColumn() : null) {
final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues();
long size = 0;

if (analyzingSize() && complexColumn != null) {
final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName);
if (serde == null) {
return ColumnAnalysis.error(String.format("unknown_complex_%s", typeName));
}

if (analyzingSize() && complexColumn != null) {
final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName);
if (serde == null) {
return ColumnAnalysis.error(String.format("unknown_complex_%s", typeName));
}
final Function<Object, Long> inputSizeFn = serde.inputSizeFn();
if (inputSizeFn == null) {
return new ColumnAnalysis(typeName, hasMultipleValues, 0, null, null, null, null);
}

final Function<Object, Long> inputSizeFn = serde.inputSizeFn();
if (inputSizeFn == null) {
return new ColumnAnalysis(typeName, hasMultipleValues, 0, null, null, null, null);
final int length = column.getLength();
for (int i = 0; i < length; ++i) {
size += inputSizeFn.apply(complexColumn.getRowValue(i));
}
}

final int length = column.getLength();
for (int i = 0; i < length; ++i) {
size += inputSizeFn.apply(complexColumn.getRowValue(i));
}
return new ColumnAnalysis(
typeName,
hasMultipleValues,
size,
null,
null,
null,
null
);
}

return new ColumnAnalysis(
typeName,
hasMultipleValues,
size,
null,
null,
null,
null
);
}
}
22 changes: 13 additions & 9 deletions processing/src/main/java/io/druid/query/search/SearchQueryRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,18 +121,22 @@ public Sequence<Result<SearchResultValue>> run(
if (!interval.contains(segment.getDataInterval())) {
MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap();
final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME);
final GenericColumn timeValues = timeColumn.getGenericColumn();
try (final GenericColumn timeValues = timeColumn.getGenericColumn()) {

int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true));
int endIndex = Math.min(timeValues.length() - 1, getStartIndexOfTime(timeValues, interval.getEndMillis(), false));
int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true));
int endIndex = Math.min(
timeValues.length() - 1,
getStartIndexOfTime(timeValues, interval.getEndMillis(), false)
);

for (int i = startIndex; i <= endIndex; i++) {
timeBitmap.add(i);
}
for (int i = startIndex; i <= endIndex; i++) {
timeBitmap.add(i);
}

final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap);
timeFilteredBitmap =
(baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter);
final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap);
timeFilteredBitmap =
(baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter);
}
} else {
timeFilteredBitmap = baseFilter;
}
Expand Down
7 changes: 1 addition & 6 deletions processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,14 +98,9 @@ public Iterator<String> iterator()
@Override
public int getNumRows()
{
GenericColumn column = null;
try {
column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();
try (final GenericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn()) {
return column.length();
}
finally {
CloseQuietly.close(column);
}
}

@Override
Expand Down
31 changes: 18 additions & 13 deletions processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Closer;
import com.metamx.common.ISE;
import com.metamx.common.guava.CloseQuietly;
import com.metamx.common.logger.Logger;
Expand Down Expand Up @@ -178,8 +179,9 @@ public Iterator<Rowboat> iterator()
return new Iterator<Rowboat>()
{
final GenericColumn timestamps = input.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();
final Object[] metrics;
final Closeable[] metrics;
final Closeable[] columns;
final Closer closer = Closer.create();

final int numMetrics = getMetricNames().size();

Expand All @@ -190,6 +192,8 @@ public Iterator<Rowboat> iterator()
boolean done = false;

{
closer.register(timestamps);

handlerSet.toArray(handlers);
this.columns = FluentIterable
.from(handlerSet)
Expand All @@ -204,9 +208,12 @@ public Closeable apply(DimensionHandler handler)
}
}
).toArray(Closeable.class);
for (Closeable column : columns) {
closer.register(column);
}

final Indexed<String> availableMetrics = getMetricNames();
metrics = new Object[availableMetrics.size()];
metrics = new Closeable[availableMetrics.size()];
for (int i = 0; i < metrics.length; ++i) {
final Column column = input.getColumn(availableMetrics.get(i));
final ValueType type = column.getCapabilities().getType();
Expand All @@ -222,22 +229,17 @@ public Closeable apply(DimensionHandler handler)
throw new ISE("Cannot handle type[%s]", type);
}
}
for (Closeable metricColumn : metrics) {
closer.register(metricColumn);
}
}

@Override
public boolean hasNext()
{
final boolean hasNext = currRow < numRows;
if (!hasNext && !done) {
CloseQuietly.close(timestamps);
for (Object metric : metrics) {
if (metric instanceof Closeable) {
CloseQuietly.close((Closeable) metric);
}
}
for (Closeable dimension : columns) {
CloseQuietly.close(dimension);
}
CloseQuietly.close(closer);
done = true;
}
return hasNext;
Expand Down Expand Up @@ -315,8 +317,11 @@ public String getMetricType(String metric)
return "float";
case LONG:
return "long";
case COMPLEX:
return column.getComplexColumn().getTypeName();
case COMPLEX: {
try (ComplexColumn complexColumn = column.getComplexColumn() ) {
return complexColumn.getTypeName();
}
}
default:
throw new ISE("Unknown type[%s]", type);
}
Expand Down
54 changes: 14 additions & 40 deletions processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.Closer;
import com.metamx.collections.bitmap.ImmutableBitmap;
import com.metamx.common.IAE;
import com.metamx.common.UOE;
import com.metamx.common.guava.CloseQuietly;
import com.metamx.common.guava.Sequence;
import com.metamx.common.guava.Sequences;
import io.druid.granularity.QueryGranularity;
Expand Down Expand Up @@ -135,27 +133,17 @@ public int getNumRows()
@Override
public DateTime getMinTime()
{
GenericColumn column = null;
try {
column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();
try (final GenericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn()) {
return new DateTime(column.getLongSingleValueRow(0));
}
finally {
CloseQuietly.close(column);
}
}

@Override
public DateTime getMaxTime()
{
GenericColumn column = null;
try {
column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();
try (final GenericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn()) {
return new DateTime(column.getLongSingleValueRow(column.length() - 1));
}
finally {
CloseQuietly.close(column);
}
}

@Override
Expand Down Expand Up @@ -202,8 +190,9 @@ public Map<String, DimensionHandler> getDimensionHandlers()
public String getColumnTypeName(String columnName)
{
final Column column = index.getColumn(columnName);
final ComplexColumn complexColumn = column.getComplexColumn();
return complexColumn != null ? complexColumn.getTypeName() : column.getCapabilities().getType().toString();
try (final ComplexColumn complexColumn = column.getComplexColumn()) {
return complexColumn != null ? complexColumn.getTypeName() : column.getCapabilities().getType().toString();
}
}

@Override
Expand Down Expand Up @@ -382,11 +371,13 @@ public Sequence<Cursor> build()

final Map<String, DictionaryEncodedColumn> dictionaryColumnCache = Maps.newHashMap();
final Map<String, GenericColumn> genericColumnCache = Maps.newHashMap();
final Map<String, ComplexColumn> complexColumnCache = Maps.newHashMap();
final Map<String, Object> objectColumnCache = Maps.newHashMap();

final GenericColumn timestamps = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();

final Closer closer = Closer.create();
closer.register(timestamps);

Iterable<Long> iterable = gran.iterable(interval.getStartMillis(), interval.getEndMillis());
if (descending) {
iterable = Lists.reverse(ImmutableList.copyOf(iterable));
Expand Down Expand Up @@ -471,6 +462,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(
DictionaryEncodedColumn<String> cachedColumn = dictionaryColumnCache.get(dimension);
if (cachedColumn == null) {
cachedColumn = columnDesc.getDictionaryEncoding();
closer.register(cachedColumn);
dictionaryColumnCache.put(dimension, cachedColumn);
}

Expand Down Expand Up @@ -592,6 +584,7 @@ public FloatColumnSelector makeFloatColumnSelector(String columnName)
if (holder != null && (holder.getCapabilities().getType() == ValueType.FLOAT
|| holder.getCapabilities().getType() == ValueType.LONG)) {
cachedMetricVals = holder.getGenericColumn();
closer.register(cachedMetricVals);
genericColumnCache.put(columnName, cachedMetricVals);
}
}
Expand Down Expand Up @@ -628,6 +621,7 @@ public LongColumnSelector makeLongColumnSelector(String columnName)
if (holder != null && (holder.getCapabilities().getType() == ValueType.LONG
|| holder.getCapabilities().getType() == ValueType.FLOAT)) {
cachedMetricVals = holder.getGenericColumn();
closer.register(cachedMetricVals);
genericColumnCache.put(columnName, cachedMetricVals);
}
}
Expand Down Expand Up @@ -676,6 +670,7 @@ public ObjectColumnSelector makeObjectColumnSelector(String column)
}

if (cachedColumnVals != null) {
closer.register((Closeable) cachedColumnVals);
objectColumnCache.put(column, cachedColumnVals);
}
}
Expand Down Expand Up @@ -953,28 +948,7 @@ public void reset()
}
}
),
new Closeable()
{
@Override
public void close() throws IOException
{
CloseQuietly.close(timestamps);
for (DictionaryEncodedColumn column : dictionaryColumnCache.values()) {
CloseQuietly.close(column);
}
for (GenericColumn column : genericColumnCache.values()) {
CloseQuietly.close(column);
}
for (ComplexColumn complexColumn : complexColumnCache.values()) {
CloseQuietly.close(complexColumn);
}
for (Object column : objectColumnCache.values()) {
if (column instanceof Closeable) {
CloseQuietly.close((Closeable) column);
}
}
}
}
closer
);
}
}
Expand Down
3 changes: 3 additions & 0 deletions processing/src/main/java/io/druid/segment/column/ComplexColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,7 @@ public interface ComplexColumn extends Closeable
public Class<?> getClazz();
public String getTypeName();
public Object getRowValue(int rowNum);

@Override
void close();
}
57 changes: 0 additions & 57 deletions processing/src/main/java/io/druid/segment/column/ComplexColumnImpl.java
View file
Open in desktop

This file was deleted.

3 changes: 3 additions & 0 deletions processing/src/main/java/io/druid/segment/column/GenericColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,7 @@ public interface GenericColumn extends Closeable
public IndexedFloats getFloatMultiValueRow(int rowNum);
public long getLongSingleValueRow(int rowNum);
public IndexedLongs getLongMultiValueRow(int rowNum);

@Override
void close();
}
Loading