Skip to content

Optimize CachingLocalSegmentAllocator#getSequenceName #8909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jon-wei merged 8 commits into from
Dec 24, 2019
Merged

Optimize CachingLocalSegmentAllocator#getSequenceName #8909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ccb03c0
Optimize CachingLocalSegmentAllocator#getSequenceName
Nov 19, 2019
7c22095
fix imports and add test
Nov 19, 2019
c04f689
Add comment about optimization
Nov 19, 2019
82a2c86
Use renamed function for TaskToolbox
Nov 20, 2019
9d29652
Move tests after refactor
suneet-s Dec 17, 2019
0f5ab3a
Merge remote-tracking branch 'upstream/master' into sequence-name-opt
suneet-s Dec 17, 2019
38c604e
Rename tests
suneet-s Dec 18, 2019
7a16688
Merge remote-tracking branch 'upstream/master' into sequence-name-opt
suneet-s Dec 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions ...c/main/java/org/apache/druid/indexing/common/task/CachingLocalSegmentAllocatorHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.apache.druid.indexing.common.actions.SurrogateAction;
import org.apache.druid.indexing.common.task.IndexTask.ShardSpecs;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
import org.apache.druid.timeline.partition.ShardSpec;
import org.joda.time.Interval;
Expand Down Expand Up @@ -133,6 +132,8 @@ public String getSequenceName(Interval interval, InputRow inputRow)
*/
private String getSequenceName(Interval interval, ShardSpec shardSpec)
{
return StringUtils.format("%s_%s_%d", taskId, interval, shardSpec.getPartitionNum());
// Note: We do not use String format here since this can be called in a tight loop
// and it's faster to add strings together than it is to use String#format
return taskId + "_" + interval + "_" + shardSpec.getPartitionNum();
}
}
12 changes: 12 additions & 0 deletions ...org/apache/druid/indexing/common/task/RangePartitionCachingLocalSegmentAllocatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.druid.indexing.common.task.batch.parallel.distribution.PartitionBoundaries;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.partition.SingleDimensionShardSpec;
Expand Down Expand Up @@ -141,6 +142,17 @@ public void allocatesCorrectShardSpecsForLastPartition()
testAllocate(row, interval, partitionNum, null);
}

@Test
public void getSequenceName()
{
// getSequenceName_forIntervalAndRow_shouldUseISOFormatAndPartitionNumForRow
Interval interval = INTERVAL_NORMAL;
InputRow row = createInputRow(interval, PARTITION9);
String sequenceName = target.getSequenceName(interval, row);
String expectedSequenceName = StringUtils.format("%s_%s_%d", TASKID, interval, 1);
Assert.assertEquals(expectedSequenceName, sequenceName);
}

@SuppressWarnings("SameParameterValue")
private void testAllocate(InputRow row, Interval interval, int partitionNum)
{
Expand Down
12 changes: 12 additions & 0 deletions ...id/indexing/common/task/batch/parallel/HashPartitionCachingLocalSegmentAllocatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec;
Expand Down Expand Up @@ -101,6 +102,17 @@ public void allocatesCorrectShardSpec() throws IOException
Assert.assertEquals(PARTITION_NUM, shardSpec.getPartitionNum());
}


@Test
public void getSequenceName()
{
// getSequenceName_forIntervalAndRow_shouldUseISOFormatAndPartitionNumForRow
InputRow row = createInputRow();
String sequenceName = target.getSequenceName(INTERVAL, row);
String expectedSequenceName = StringUtils.format("%s_%s_%d", TASKID, INTERVAL, PARTITION_NUM);
Assert.assertEquals(expectedSequenceName, sequenceName);
}

private static TaskToolbox createToolbox()
{
TaskToolbox toolbox = EasyMock.mock(TaskToolbox.class);
Expand Down