From f54581e0257c7d75e648188775ab60d583434aed Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 3 Aug 2016 22:59:29 -0500 Subject: [PATCH] optionally dump list of segment-ids used to process the query --- .../io/druid/query/SegmentDescriptor.java | 11 ++++++ .../java/io/druid/timeline/DataSegment.java | 39 +++++++++++++++---- .../io/druid/timeline/DataSegmentUtils.java | 6 +-- .../main/java/io/druid/query/BaseQuery.java | 5 +++ .../druid/client/CachingClusteredClient.java | 9 +++++ 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/io/druid/query/SegmentDescriptor.java b/api/src/main/java/io/druid/query/SegmentDescriptor.java index cfad1e082980..cad76939c4c6 100644 --- a/api/src/main/java/io/druid/query/SegmentDescriptor.java +++ b/api/src/main/java/io/druid/query/SegmentDescriptor.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; /** @@ -86,6 +87,16 @@ public boolean equals(Object o) return true; } + public String getSegmentIdWithoutDataSource() + { + return DataSegment.makeDataSegmentIdentifierWithoutDataSource( + interval.getStart(), + interval.getEnd(), + version, + partitionNumber + ); + } + @Override public int hashCode() { diff --git a/api/src/main/java/io/druid/timeline/DataSegment.java b/api/src/main/java/io/druid/timeline/DataSegment.java index 5b874d622049..1ceef08b7f7f 100644 --- a/api/src/main/java/io/druid/timeline/DataSegment.java +++ b/api/src/main/java/io/druid/timeline/DataSegment.java @@ -48,7 +48,7 @@ */ public class DataSegment implements Comparable { - public static String delimiter = "_"; + public final static String DELIMITER = "_"; private final Integer binaryVersion; private static final Interner interner = Interners.newWeakInterner(); private static final Function internFun = new Function() @@ -70,18 +70,43 @@ public static String makeDataSegmentIdentifier( { StringBuilder sb = new StringBuilder(); - sb.append(dataSource).append(delimiter) - .append(start).append(delimiter) - .append(end).append(delimiter) + sb.append(dataSource).append(DELIMITER); + updateBuilderWithSegmentIdWithoutDataSource(sb, start, end, version, shardSpec.getPartitionNum()); + + return sb.toString(); + } + + private static void updateBuilderWithSegmentIdWithoutDataSource( + StringBuilder sb, + DateTime start, + DateTime end, + String version, + int partitionNum + ) + { + sb.append(start).append(DELIMITER) + .append(end).append(DELIMITER) .append(version); - if (shardSpec.getPartitionNum() != 0) { - sb.append(delimiter).append(shardSpec.getPartitionNum()); + if (partitionNum != 0) { + sb.append(DELIMITER).append(partitionNum); } + } + + public static String makeDataSegmentIdentifierWithoutDataSource( + DateTime start, + DateTime end, + String version, + int partitionNum + ) + { + StringBuilder sb = new StringBuilder(); + + updateBuilderWithSegmentIdWithoutDataSource(sb, start, end, version, partitionNum); return sb.toString(); } - + private final String dataSource; private final Interval interval; private final String version; diff --git a/api/src/main/java/io/druid/timeline/DataSegmentUtils.java b/api/src/main/java/io/druid/timeline/DataSegmentUtils.java index d2b8a6888c5c..4521208d8bb1 100644 --- a/api/src/main/java/io/druid/timeline/DataSegmentUtils.java +++ b/api/src/main/java/io/druid/timeline/DataSegmentUtils.java @@ -64,7 +64,7 @@ private static SegmentIdentifierParts parse(String dataSource, String identifier return null; } String remaining = identifier.substring(dataSource.length() + 1); - String[] splits = remaining.split(DataSegment.delimiter); + String[] splits = remaining.split(DataSegment.DELIMITER); if (splits.length < 3) { LOGGER.info("Invalid identifier %s", identifier); return null; @@ -74,7 +74,7 @@ private static SegmentIdentifierParts parse(String dataSource, String identifier DateTime start = formatter.parseDateTime(splits[0]); DateTime end = formatter.parseDateTime(splits[1]); String version = splits[2]; - String trail = splits.length > 3 ? join(splits, DataSegment.delimiter, 3, splits.length) : null; + String trail = splits.length > 3 ? join(splits, DataSegment.DELIMITER, 3, splits.length) : null; return new SegmentIdentifierParts( dataSource, @@ -169,7 +169,7 @@ public String toString() { return join( new Object[]{dataSource, interval.getStart(), interval.getEnd(), version, trail}, - DataSegment.delimiter, 0, version == null ? 3 : trail == null ? 4 : 5 + DataSegment.DELIMITER, 0, version == null ? 3 : trail == null ? 4 : 5 ); } } diff --git a/processing/src/main/java/io/druid/query/BaseQuery.java b/processing/src/main/java/io/druid/query/BaseQuery.java index 69f297a48618..3bc9768ced0d 100644 --- a/processing/src/main/java/io/druid/query/BaseQuery.java +++ b/processing/src/main/java/io/druid/query/BaseQuery.java @@ -67,6 +67,11 @@ public static int getContextUncoveredIntervalsLimit(Query query, int defa return parseInt(query, "uncoveredIntervalsLimit", defaultValue); } + public static boolean getContextDumpSegmentList(Query query, boolean defaultValue) + { + return parseBoolean(query, "dumpSegmentList", defaultValue); + } + private static int parseInt(Query query, String key, int defaultValue) { Object val = query.getContextValue(key); diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index 66dfeb524c3d..8a12fda28509 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -248,6 +248,15 @@ public ShardSpec apply(PartitionChunk input) } } + // Optionally dump all used segmentIds in processing this query + if (BaseQuery.getContextDumpSegmentList(query, false)) { + List segmentIds = new ArrayList<>(segments.size()); + for (Pair segment : segments) { + segmentIds.add(segment.rhs.getSegmentIdWithoutDataSource()); + } + responseContext.put("segments", segmentIds); + } + final byte[] queryCacheKey; if ((populateCache || useCache) // implies strategy != null