diff --git a/core/src/main/java/org/apache/druid/java/util/common/granularity/AllGranularity.java b/core/src/main/java/org/apache/druid/java/util/common/granularity/AllGranularity.java index 7299bf5281c3..01ed140376b8 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/granularity/AllGranularity.java +++ b/core/src/main/java/org/apache/druid/java/util/common/granularity/AllGranularity.java @@ -62,6 +62,15 @@ public DateTime toDate(String filePath, Formatter formatter) throw new UnsupportedOperationException("This method should not be invoked for this granularity type"); } + /** + * No interval is aligned with all granularity since it's infinite. + */ + @Override + public boolean isAligned(Interval interval) + { + return false; + } + @Override public byte[] getCacheKey() { diff --git a/core/src/main/java/org/apache/druid/java/util/common/granularity/DurationGranularity.java b/core/src/main/java/org/apache/druid/java/util/common/granularity/DurationGranularity.java index a48c071b5d22..d9af7eda819d 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/granularity/DurationGranularity.java +++ b/core/src/main/java/org/apache/druid/java/util/common/granularity/DurationGranularity.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import org.apache.druid.java.util.common.DateTimes; import org.joda.time.DateTime; +import org.joda.time.Interval; import org.joda.time.format.DateTimeFormatter; import java.nio.ByteBuffer; @@ -94,6 +95,15 @@ public DateTime toDate(String filePath, Formatter formatter) throw new UnsupportedOperationException("This method should not be invoked for this granularity type"); } + @Override + public boolean isAligned(Interval interval) + { + if (interval.toDurationMillis() == duration) { + return (interval.getStartMillis() - origin) % duration == 0; + } + return false; + } + @Override public byte[] getCacheKey() { diff --git a/core/src/main/java/org/apache/druid/java/util/common/granularity/Granularity.java b/core/src/main/java/org/apache/druid/java/util/common/granularity/Granularity.java index e2b7c33e5780..3a9cdac17b73 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/granularity/Granularity.java +++ b/core/src/main/java/org/apache/druid/java/util/common/granularity/Granularity.java @@ -113,6 +113,11 @@ public static List granularitiesFinerThan(final Granularity gran0) public abstract DateTime toDate(String filePath, Formatter formatter); + /** + * Return true if time chunks populated by this granularity includes the given interval time chunk. + */ + public abstract boolean isAligned(Interval interval); + public DateTime bucketEnd(DateTime time) { return increment(bucketStart(time)); diff --git a/core/src/main/java/org/apache/druid/java/util/common/granularity/NoneGranularity.java b/core/src/main/java/org/apache/druid/java/util/common/granularity/NoneGranularity.java index 06c65b01269d..750ea2a335e1 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/granularity/NoneGranularity.java +++ b/core/src/main/java/org/apache/druid/java/util/common/granularity/NoneGranularity.java @@ -20,6 +20,7 @@ package org.apache.druid.java.util.common.granularity; import org.joda.time.DateTime; +import org.joda.time.Interval; import org.joda.time.format.DateTimeFormatter; /** @@ -59,6 +60,15 @@ public DateTime toDate(String filePath, Formatter formatter) throw new UnsupportedOperationException("This method should not be invoked for this granularity type"); } + /** + * Any interval is aligned with none granularity since it's effectively millisecond granularity. + */ + @Override + public boolean isAligned(Interval interval) + { + return true; + } + @Override public byte[] getCacheKey() { diff --git a/core/src/main/java/org/apache/druid/java/util/common/granularity/PeriodGranularity.java b/core/src/main/java/org/apache/druid/java/util/common/granularity/PeriodGranularity.java index a88d53088ad1..b1f36e6c6724 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/granularity/PeriodGranularity.java +++ b/core/src/main/java/org/apache/druid/java/util/common/granularity/PeriodGranularity.java @@ -32,6 +32,7 @@ import org.joda.time.Chronology; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.joda.time.Interval; import org.joda.time.Period; import org.joda.time.chrono.ISOChronology; import org.joda.time.format.DateTimeFormat; @@ -136,6 +137,12 @@ public DateTime toDate(String filePath, Formatter formatter) return null; } + @Override + public boolean isAligned(Interval interval) + { + return bucket(interval.getStart()).equals(interval); + } + @Override public byte[] getCacheKey() { diff --git a/core/src/main/java/org/apache/druid/timeline/DataSegment.java b/core/src/main/java/org/apache/druid/timeline/DataSegment.java index fb6a287fc3a5..c427f6a04ea6 100644 --- a/core/src/main/java/org/apache/druid/timeline/DataSegment.java +++ b/core/src/main/java/org/apache/druid/timeline/DataSegment.java @@ -56,7 +56,7 @@ * {@link SegmentId} of the segment. */ @PublicApi -public class DataSegment implements Comparable +public class DataSegment implements Comparable, Overshadowable { /* * The difference between this class and org.apache.druid.segment.Segment is that this class contains the segment @@ -93,6 +93,29 @@ public static class PruneLoadSpecHolder private final ShardSpec shardSpec; private final long size; + public DataSegment( + SegmentId segmentId, + Map loadSpec, + List dimensions, + List metrics, + ShardSpec shardSpec, + Integer binaryVersion, + long size + ) + { + this( + segmentId.getDataSource(), + segmentId.getInterval(), + segmentId.getVersion(), + loadSpec, + dimensions, + metrics, + shardSpec, + binaryVersion, + size + ); + } + public DataSegment( String dataSource, Interval interval, @@ -206,7 +229,8 @@ public Map getLoadSpec() return loadSpec; } - @JsonProperty + @JsonProperty("version") + @Override public String getVersion() { return id.getVersion(); @@ -251,6 +275,50 @@ public SegmentId getId() return id; } + @Override + public boolean overshadows(DataSegment other) + { + if (id.getDataSource().equals(other.id.getDataSource()) && id.getInterval().overlaps(other.id.getInterval())) { + final int majorVersionCompare = id.getVersion().compareTo(other.id.getVersion()); + if (majorVersionCompare > 0) { + return true; + } else if (majorVersionCompare == 0) { + return includeRootPartitions(other) && getMinorVersion() > other.getMinorVersion(); + } + } + return false; + } + + @Override + public int getStartRootPartitionId() + { + return shardSpec.getStartRootPartitionId(); + } + + @Override + public int getEndRootPartitionId() + { + return shardSpec.getEndRootPartitionId(); + } + + @Override + public short getMinorVersion() + { + return shardSpec.getMinorVersion(); + } + + @Override + public short getAtomicUpdateGroupSize() + { + return shardSpec.getAtomicUpdateGroupSize(); + } + + private boolean includeRootPartitions(DataSegment other) + { + return shardSpec.getStartRootPartitionId() <= other.shardSpec.getStartRootPartitionId() + && shardSpec.getEndRootPartitionId() >= other.shardSpec.getEndRootPartitionId(); + } + public SegmentDescriptor toDescriptor() { return id.toDescriptor(); @@ -271,6 +339,11 @@ public DataSegment withMetrics(List metrics) return builder(this).metrics(metrics).build(); } + public DataSegment withShardSpec(ShardSpec newSpec) + { + return builder(this).shardSpec(newSpec).build(); + } + public DataSegment withSize(long size) { return builder(this).size(size).build(); @@ -311,15 +384,13 @@ public int hashCode() public String toString() { return "DataSegment{" + - "size=" + size + - ", shardSpec=" + shardSpec + - ", metrics=" + metrics + - ", dimensions=" + dimensions + - ", version='" + getVersion() + '\'' + + "binaryVersion=" + binaryVersion + + ", id=" + id + ", loadSpec=" + loadSpec + - ", interval=" + getInterval() + - ", dataSource='" + getDataSource() + '\'' + - ", binaryVersion='" + binaryVersion + '\'' + + ", dimensions=" + dimensions + + ", metrics=" + metrics + + ", shardSpec=" + shardSpec + + ", size=" + size + '}'; } diff --git a/core/src/main/java/org/apache/druid/timeline/Overshadowable.java b/core/src/main/java/org/apache/druid/timeline/Overshadowable.java new file mode 100644 index 000000000000..69b4336f459a --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/Overshadowable.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline; + +/** + * Interface to represent a class which can have overshadow relation between its instances. + * In {@link VersionedIntervalTimeline}, Overshadowable is used to represent each {@link DataSegment} + * which has the same major version in the same time chunk. + * + * An Overshadowable overshadows another if its root partition range contains that of another + * and has a higher minorVersion. For more details, check https://github.com/apache/incubator-druid/issues/7491. + */ +public interface Overshadowable +{ + /** + * Returns true if this overshadowable overshadows the other. + */ + default boolean overshadows(T other) + { + final int majorVersionCompare = getVersion().compareTo(other.getVersion()); + if (majorVersionCompare == 0) { + return containsRootPartition(other) && getMinorVersion() > other.getMinorVersion(); + } else { + return majorVersionCompare > 0; + } + } + + default boolean containsRootPartition(T other) + { + return getStartRootPartitionId() <= other.getStartRootPartitionId() + && getEndRootPartitionId() >= other.getEndRootPartitionId(); + } + + /** + * All overshadowables have root partition range. + * First-generation overshadowables have (partitionId, partitionId + 1) as their root partition range. + * Non-first-generation overshadowables are the overshadowables that overwrite first or non-first generation + * overshadowables, and they have the merged root partition range of all overwritten first-generation overshadowables. + * + * Note that first-generation overshadowables can be overwritten by a single non-first-generation overshadowable + * if they have consecutive partitionId. Non-first-generation overshadowables can be overwritten by another + * if their root partition ranges are consecutive. + */ + int getStartRootPartitionId(); + + /** + * See doc of {@link #getStartRootPartitionId()}. + */ + int getEndRootPartitionId(); + + String getVersion(); + + short getMinorVersion(); + + /** + * Return the size of atomicUpdateGroup. + * An atomicUpdateGroup is a set of segments which should be updated all together atomically in + * {@link VersionedIntervalTimeline}. + */ + short getAtomicUpdateGroupSize(); +} diff --git a/core/src/main/java/org/apache/druid/timeline/SegmentId.java b/core/src/main/java/org/apache/druid/timeline/SegmentId.java index 8ac01e4e31a1..37fd68d4603a 100644 --- a/core/src/main/java/org/apache/druid/timeline/SegmentId.java +++ b/core/src/main/java/org/apache/druid/timeline/SegmentId.java @@ -323,7 +323,6 @@ public Interval getInterval() return new Interval(intervalStartMillis, intervalEndMillis, intervalChronology); } - @Nullable public String getVersion() { return version; diff --git a/core/src/main/java/org/apache/druid/timeline/TimelineLookup.java b/core/src/main/java/org/apache/druid/timeline/TimelineLookup.java index 6bdab5c723de..fb9e577b83cf 100644 --- a/core/src/main/java/org/apache/druid/timeline/TimelineLookup.java +++ b/core/src/main/java/org/apache/druid/timeline/TimelineLookup.java @@ -26,7 +26,7 @@ import java.util.List; -public interface TimelineLookup +public interface TimelineLookup> { /** diff --git a/core/src/main/java/org/apache/druid/timeline/TimelineObjectHolder.java b/core/src/main/java/org/apache/druid/timeline/TimelineObjectHolder.java index 3feca88495c3..4aa1738b2aca 100644 --- a/core/src/main/java/org/apache/druid/timeline/TimelineObjectHolder.java +++ b/core/src/main/java/org/apache/druid/timeline/TimelineObjectHolder.java @@ -23,9 +23,11 @@ import org.apache.druid.timeline.partition.PartitionHolder; import org.joda.time.Interval; +import java.util.Objects; + /** */ -public class TimelineObjectHolder implements LogicalSegment +public class TimelineObjectHolder> implements LogicalSegment { private final Interval interval; private final Interval trueInterval; @@ -73,6 +75,27 @@ public PartitionHolder getObject() return object; } + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TimelineObjectHolder that = (TimelineObjectHolder) o; + return Objects.equals(interval, that.interval) && + Objects.equals(version, that.version) && + Objects.equals(object, that.object); + } + + @Override + public int hashCode() + { + return Objects.hash(interval, version, object); + } + @Override public String toString() { diff --git a/core/src/main/java/org/apache/druid/timeline/VersionedIntervalTimeline.java b/core/src/main/java/org/apache/druid/timeline/VersionedIntervalTimeline.java index 8deed36fd7e6..4998c41c0f65 100644 --- a/core/src/main/java/org/apache/druid/timeline/VersionedIntervalTimeline.java +++ b/core/src/main/java/org/apache/druid/timeline/VersionedIntervalTimeline.java @@ -42,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.NavigableMap; import java.util.Objects; import java.util.Set; @@ -63,21 +64,27 @@ * a certain time period and when you do a lookup(), you are asking for all of the objects that you need to look * at in order to get a correct answer about that time period. * - * The findOvershadowed() method returns a list of objects that will never be returned by a call to lookup() because + * The findFullyOvershadowed() method returns a list of objects that will never be returned by a call to lookup() because * they are overshadowed by some other object. This can be used in conjunction with the add() and remove() methods * to achieve "atomic" updates. First add new items, then check if those items caused anything to be overshadowed, if * so, remove the overshadowed elements and you have effectively updated your data set without any user impact. */ -public class VersionedIntervalTimeline implements TimelineLookup +public class VersionedIntervalTimeline> implements TimelineLookup { private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); - final NavigableMap completePartitionsTimeline = new TreeMap( + // Below timelines stores only *visible* timelineEntries + // adjusted interval -> timelineEntry + private final NavigableMap completePartitionsTimeline = new TreeMap<>( Comparators.intervalsByStartThenEnd() ); - final NavigableMap incompletePartitionsTimeline = new TreeMap( + // IncompletePartitionsTimeline also includes completePartitionsTimeline + // adjusted interval -> timelineEntry + @VisibleForTesting + final NavigableMap incompletePartitionsTimeline = new TreeMap<>( Comparators.intervalsByStartThenEnd() ); + // true interval -> version -> timelineEntry private final Map> allTimelineEntries = new HashMap<>(); private final AtomicInteger numObjects = new AtomicInteger(); @@ -113,14 +120,13 @@ public static void addSegments( ); } - @VisibleForTesting public Map> getAllTimelineEntries() { return allTimelineEntries; } /** - * Returns a lazy collection with all objects (including overshadowed, see {@link #findOvershadowed}) in this + * Returns a lazy collection with all objects (including overshadowed, see {@link #findFullyOvershadowed}) in this * VersionedIntervalTimeline to be used for iteration or {@link Collection#stream()} transformation. The order of * objects in this collection is unspecified. * @@ -189,7 +195,7 @@ private void addAll( } // "isComplete" is O(objects in holder) so defer it to the end of addAll. - for (Map.Entry entry : allEntries.entrySet()) { + for (Entry entry : allEntries.entrySet()) { Interval interval = entry.getValue(); if (entry.getKey().getPartitionHolder().isComplete()) { @@ -207,9 +213,8 @@ private void addAll( @Nullable public PartitionChunk remove(Interval interval, VersionType version, PartitionChunk chunk) { + lock.writeLock().lock(); try { - lock.writeLock().lock(); - Map versionEntries = allTimelineEntries.get(interval); if (versionEntries == null) { return null; @@ -246,9 +251,9 @@ public PartitionChunk remove(Interval interval, VersionType version, @Override public @Nullable PartitionHolder findEntry(Interval interval, VersionType version) { + lock.readLock().lock(); try { - lock.readLock().lock(); - for (Map.Entry> entry : allTimelineEntries.entrySet()) { + for (Entry> entry : allTimelineEntries.entrySet()) { if (entry.getKey().equals(interval) || entry.getKey().contains(interval)) { TimelineEntry foundEntry = entry.getValue().get(version); if (foundEntry != null) { @@ -276,8 +281,8 @@ public PartitionChunk remove(Interval interval, VersionType version, @Override public List> lookup(Interval interval) { + lock.readLock().lock(); try { - lock.readLock().lock(); return lookup(interval, false); } finally { @@ -288,8 +293,8 @@ public List> lookup(Interval inter @Override public List> lookupWithIncompletePartitions(Interval interval) { + lock.readLock().lock(); try { - lock.readLock().lock(); return lookup(interval, true); } finally { @@ -299,8 +304,8 @@ public List> lookupWithIncompleteP public boolean isEmpty() { + lock.readLock().lock(); try { - lock.readLock().lock(); return completePartitionsTimeline.isEmpty(); } finally { @@ -310,8 +315,8 @@ public boolean isEmpty() public TimelineObjectHolder first() { + lock.readLock().lock(); try { - lock.readLock().lock(); return timelineEntryToObjectHolder(completePartitionsTimeline.firstEntry().getValue()); } finally { @@ -321,8 +326,8 @@ public TimelineObjectHolder first() public TimelineObjectHolder last() { + lock.readLock().lock(); try { - lock.readLock().lock(); return timelineEntryToObjectHolder(completePartitionsTimeline.lastEntry().getValue()); } finally { @@ -344,20 +349,19 @@ private TimelineObjectHolder timelineEntryToObjectHolde * This method should be deduplicated with DataSourcesSnapshot.determineOvershadowedSegments(): see * https://github.com/apache/incubator-druid/issues/8070. */ - public Set> findOvershadowed() + public Set> findFullyOvershadowed() { + lock.readLock().lock(); try { - lock.readLock().lock(); - Set> retVal = new HashSet<>(); - - Map> overShadowed = new HashMap<>(); + // 1. Put all timelineEntries and remove all visible entries to find out only non-visible timelineEntries. + final Map> overShadowed = new HashMap<>(); for (Map.Entry> versionEntry : allTimelineEntries.entrySet()) { @SuppressWarnings("unchecked") Map versionCopy = (TreeMap) versionEntry.getValue().clone(); overShadowed.put(versionEntry.getKey(), versionCopy); } - for (Map.Entry entry : completePartitionsTimeline.entrySet()) { + for (Entry entry : completePartitionsTimeline.entrySet()) { Map versionEntry = overShadowed.get(entry.getValue().getTrueInterval()); if (versionEntry != null) { versionEntry.remove(entry.getValue().getVersion()); @@ -367,7 +371,7 @@ public Set> findOvershadowed() } } - for (Map.Entry entry : incompletePartitionsTimeline.entrySet()) { + for (Entry entry : incompletePartitionsTimeline.entrySet()) { Map versionEntry = overShadowed.get(entry.getValue().getTrueInterval()); if (versionEntry != null) { versionEntry.remove(entry.getValue().getVersion()); @@ -377,10 +381,25 @@ public Set> findOvershadowed() } } - for (Map.Entry> versionEntry : overShadowed.entrySet()) { - for (Map.Entry entry : versionEntry.getValue().entrySet()) { - TimelineEntry object = entry.getValue(); - retVal.add(timelineEntryToObjectHolder(object)); + final Set> retVal = new HashSet<>(); + for (Entry> versionEntry : overShadowed.entrySet()) { + for (Entry entry : versionEntry.getValue().entrySet()) { + final TimelineEntry timelineEntry = entry.getValue(); + retVal.add(timelineEntryToObjectHolder(timelineEntry)); + } + } + + // 2. Visible timelineEntries can also have overshadowed segments. Add them to the result too. + for (TimelineEntry entry : incompletePartitionsTimeline.values()) { + final List> entryOvershadowed = entry.partitionHolder.getOvershadowed(); + if (!entryOvershadowed.isEmpty()) { + retVal.add( + new TimelineObjectHolder<>( + entry.trueInterval, + entry.version, + new PartitionHolder<>(entryOvershadowed) + ) + ); } } @@ -391,14 +410,23 @@ public Set> findOvershadowed() } } - public boolean isOvershadowed(Interval interval, VersionType version) + public boolean isOvershadowed(Interval interval, VersionType version, ObjectType object) { + lock.readLock().lock(); try { - lock.readLock().lock(); - TimelineEntry entry = completePartitionsTimeline.get(interval); if (entry != null) { - return versionComparator.compare(version, entry.getVersion()) < 0; + final int majorVersionCompare = versionComparator.compare(version, entry.getVersion()); + if (majorVersionCompare == 0) { + for (PartitionChunk chunk : entry.partitionHolder) { + if (chunk.getObject().overshadows(object)) { + return true; + } + } + return false; + } else { + return majorVersionCompare < 0; + } } Interval lower = completePartitionsTimeline.floorKey( @@ -414,13 +442,23 @@ public boolean isOvershadowed(Interval interval, VersionType version) do { if (curr == null || //no further keys - (prev != null && curr.getStartMillis() > prev.getEndMillis()) || //a discontinuity - //lower or same version - versionComparator.compare(version, completePartitionsTimeline.get(curr).getVersion()) >= 0 - ) { + (prev != null && curr.getStartMillis() > prev.getEndMillis()) //a discontinuity + ) { return false; } + final TimelineEntry timelineEntry = completePartitionsTimeline.get(curr); + final int versionCompare = versionComparator.compare(version, timelineEntry.getVersion()); + + //lower or same version + if (versionCompare > 0) { + return false; + } else if (versionCompare == 0) { + if (timelineEntry.partitionHolder.stream().noneMatch(chunk -> chunk.getObject().overshadows(object))) { + return false; + } + } + prev = curr; curr = completePartitionsTimeline.higherKey(curr); @@ -490,39 +528,64 @@ private boolean addAtKey( } while (entryInterval != null && currKey != null && currKey.overlaps(entryInterval)) { - Interval nextKey = timeline.higherKey(currKey); + final Interval nextKey = timeline.higherKey(currKey); - int versionCompare = versionComparator.compare( + final int versionCompare = versionComparator.compare( entry.getVersion(), timeline.get(currKey).getVersion() ); if (versionCompare < 0) { + // since the entry version is lower than the existing one, the existing one overwrites the given entry + // if overlapped. if (currKey.contains(entryInterval)) { + // the version of the entry of currKey is larger than that of the given entry. Discard it return true; } else if (currKey.getStart().isBefore(entryInterval.getStart())) { + // | entry | + // | cur | + // => |new| entryInterval = new Interval(currKey.getEnd(), entryInterval.getEnd()); } else { + // | entry | + // | cur | + // => |new| addIntervalToTimeline(new Interval(entryInterval.getStart(), currKey.getStart()), entry, timeline); + // | entry | + // | cur | + // => |new| if (entryInterval.getEnd().isAfter(currKey.getEnd())) { entryInterval = new Interval(currKey.getEnd(), entryInterval.getEnd()); } else { - entryInterval = null; // discard this entry + // Discard this entry since there is no portion of the entry interval that goes past the end of the curr + // key interval. + entryInterval = null; } } } else if (versionCompare > 0) { - TimelineEntry oldEntry = timeline.remove(currKey); + // since the entry version is greater than the existing one, the given entry overwrites the existing one + // if overlapped. + final TimelineEntry oldEntry = timeline.remove(currKey); if (currKey.contains(entryInterval)) { + // | cur | + // | entry | + // => |old| new |old| addIntervalToTimeline(new Interval(currKey.getStart(), entryInterval.getStart()), oldEntry, timeline); addIntervalToTimeline(new Interval(entryInterval.getEnd(), currKey.getEnd()), oldEntry, timeline); addIntervalToTimeline(entryInterval, entry, timeline); return true; } else if (currKey.getStart().isBefore(entryInterval.getStart())) { + // | cur | + // | entry | + // => |old| addIntervalToTimeline(new Interval(currKey.getStart(), entryInterval.getStart()), oldEntry, timeline); } else if (entryInterval.getEnd().isBefore(currKey.getEnd())) { + // | cur | + // | entry | + // => |old| addIntervalToTimeline(new Interval(entryInterval.getEnd(), currKey.getEnd()), oldEntry, timeline); } } else { @@ -570,9 +633,9 @@ private void remove( TimelineEntry removed = timeline.get(interval); if (removed == null) { - Iterator> iter = timeline.entrySet().iterator(); + Iterator> iter = timeline.entrySet().iterator(); while (iter.hasNext()) { - Map.Entry timelineEntry = iter.next(); + Entry timelineEntry = iter.next(); if (timelineEntry.getValue() == entry) { intervalsToRemove.add(timelineEntry.getKey()); } @@ -594,7 +657,7 @@ private void remove( { timeline.remove(interval); - for (Map.Entry> versionEntry : allTimelineEntries.entrySet()) { + for (Entry> versionEntry : allTimelineEntries.entrySet()) { if (versionEntry.getKey().overlap(interval) != null) { if (incompleteOk) { add(timeline, versionEntry.getKey(), versionEntry.getValue().lastEntry().getValue()); @@ -613,12 +676,12 @@ private void remove( private List> lookup(Interval interval, boolean incompleteOk) { - List> retVal = new ArrayList>(); + List> retVal = new ArrayList<>(); NavigableMap timeline = (incompleteOk) ? incompletePartitionsTimeline : completePartitionsTimeline; - for (Map.Entry entry : timeline.entrySet()) { + for (Entry entry : timeline.entrySet()) { Interval timelineInterval = entry.getKey(); TimelineEntry val = entry.getValue(); diff --git a/core/src/main/java/org/apache/druid/timeline/partition/AtomicUpdateGroup.java b/core/src/main/java/org/apache/druid/timeline/partition/AtomicUpdateGroup.java new file mode 100644 index 000000000000..aed7f15695f1 --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/AtomicUpdateGroup.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.timeline.Overshadowable; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * A set of {@link PartitionChunk}s which should be atomically visible or not in the timeline. + * This is usually a set of single partitionChunk for first-generation segments. + * For non-first-generation segments generated by overwriting tasks, segments of the same interval generated by + * the same task become an atomicUpdateGroup. As a result, all segments in an atomicUpdateGroup have the same + * rootPartitionp range, majorVersion, minorVersion, and atomicUpdateGroupSize. + */ +class AtomicUpdateGroup> implements Overshadowable> +{ + // Perhaps it would be worth to store these in a map of (partitionId -> partitionChunk) + // because sometimes we need to search for a particular partitionChunk corresponding to a partitionId. + // However, there's a tradeoff between time and space. Storing in a map would be faster than storing in a list, + // but it would take at least additional 4 bytes per chunk to store its key. + // This may matter if there are a lot of segments to keep in memory as in brokers or the coordinator. + private final List> chunks = new ArrayList<>(); + + public AtomicUpdateGroup(PartitionChunk chunk) + { + this.chunks.add(chunk); + } + + public void add(PartitionChunk chunk) + { + if (isFull()) { + throw new IAE("Can't add more chunk[%s] to atomicUpdateGroup[%s]", chunk, chunks); + } + if (!isEmpty() && !isSameAtomicUpdateGroup(chunks.get(0), chunk)) { + throw new IAE("Can't add chunk[%s] to a different atomicUpdateGroup[%s]", chunk, chunks); + } + for (PartitionChunk existing : chunks) { + if (existing.equals(chunk)) { + return; + } + } + chunks.add(chunk); + } + + public void remove(PartitionChunk chunk) + { + if (chunks.isEmpty()) { + throw new ISE("Can't remove chunk[%s] from empty atomicUpdateGroup", chunk); + } + + if (!isSameAtomicUpdateGroup(chunks.get(0), chunk)) { + throw new IAE("Can't remove chunk[%s] from a different atomicUpdateGroup[%s]", chunk, chunks); + } + + chunks.remove(chunk); + } + + public boolean isFull() + { + return !isEmpty() && chunks.size() == chunks.get(0).getObject().getAtomicUpdateGroupSize(); + } + + public boolean isEmpty() + { + return chunks.isEmpty(); + } + + public List> getChunks() + { + return chunks; + } + + @Nullable + public PartitionChunk findChunk(int partitionId) + { + return chunks.stream().filter(chunk -> chunk.getChunkNumber() == partitionId).findFirst().orElse(null); + } + + @Override + public int getStartRootPartitionId() + { + Preconditions.checkState(!isEmpty(), "Empty atomicUpdateGroup"); + return chunks.get(0).getObject().getStartRootPartitionId(); + } + + @Override + public int getEndRootPartitionId() + { + Preconditions.checkState(!isEmpty(), "Empty atomicUpdateGroup"); + return chunks.get(0).getObject().getEndRootPartitionId(); + } + + @Override + public String getVersion() + { + Preconditions.checkState(!isEmpty(), "Empty atomicUpdateGroup"); + return chunks.get(0).getObject().getVersion(); + } + + @Override + public short getMinorVersion() + { + Preconditions.checkState(!isEmpty(), "Empty atomicUpdateGroup"); + return chunks.get(0).getObject().getMinorVersion(); + } + + @Override + public short getAtomicUpdateGroupSize() + { + Preconditions.checkState(!isEmpty(), "Empty atomicUpdateGroup"); + return chunks.get(0).getObject().getAtomicUpdateGroupSize(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AtomicUpdateGroup that = (AtomicUpdateGroup) o; + return Objects.equals(chunks, that.chunks); + } + + @Override + public int hashCode() + { + return Objects.hash(chunks); + } + + @Override + public String toString() + { + return "AtomicUpdateGroup{" + + "chunks=" + chunks + + '}'; + } + + private static > boolean isSameAtomicUpdateGroup( + PartitionChunk c1, + PartitionChunk c2 + ) + { + return c1.getObject().getStartRootPartitionId() == c2.getObject().getStartRootPartitionId() + && c1.getObject().getEndRootPartitionId() == c2.getObject().getEndRootPartitionId() + && c1.getObject().getMinorVersion() == c2.getObject().getMinorVersion() + && c1.getObject().getAtomicUpdateGroupSize() == c2.getObject().getAtomicUpdateGroupSize(); + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpec.java index 0d85423bb322..88dfc8397d25 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpec.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpec.java @@ -47,8 +47,8 @@ public class HashBasedNumberedShardSpec extends NumberedShardSpec @JsonCreator public HashBasedNumberedShardSpec( - @JsonProperty("partitionNum") int partitionNum, - @JsonProperty("partitions") int partitions, + @JsonProperty("partitionNum") int partitionNum, // partitionId + @JsonProperty("partitions") int partitions, // # of partitions @JsonProperty("partitionDimensions") @Nullable List partitionDimensions, @JacksonInject ObjectMapper jsonMapper ) @@ -64,6 +64,12 @@ public List getPartitionDimensions() return partitionDimensions; } + @Override + public boolean isCompatible(Class other) + { + return other == HashBasedNumberedShardSpec.class; + } + @Override public boolean isInChunk(long timestamp, InputRow inputRow) { diff --git a/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecFactory.java b/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecFactory.java new file mode 100644 index 000000000000..eaaa2c7085ec --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecFactory.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; + +public class HashBasedNumberedShardSpecFactory implements ShardSpecFactory +{ + @Nullable + private final List partitionDimensions; + private final int numPartitions; + + @JsonCreator + public HashBasedNumberedShardSpecFactory( + @JsonProperty("partitionDimensions") @Nullable List partitionDimensions, + @JsonProperty("numPartitions") int numPartitions + ) + { + this.partitionDimensions = partitionDimensions; + this.numPartitions = numPartitions; + } + + @Nullable + @JsonProperty + public List getPartitionDimensions() + { + return partitionDimensions; + } + + @JsonProperty public int getNumPartitions() + { + return numPartitions; + } + + @Override + public ShardSpec create(ObjectMapper objectMapper, @Nullable ShardSpec specOfPreviousMaxPartitionId) + { + final HashBasedNumberedShardSpec prevSpec = (HashBasedNumberedShardSpec) specOfPreviousMaxPartitionId; + return new HashBasedNumberedShardSpec( + prevSpec == null ? 0 : prevSpec.getPartitionNum() + 1, + numPartitions, + partitionDimensions, + objectMapper + ); + } + + @Override + public ShardSpec create(ObjectMapper objectMapper, int partitionId) + { + return new HashBasedNumberedShardSpec(partitionId, numPartitions, partitionDimensions, objectMapper); + } + + @Override + public Class getShardSpecClass() + { + return HashBasedNumberedShardSpec.class; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + HashBasedNumberedShardSpecFactory that = (HashBasedNumberedShardSpecFactory) o; + return numPartitions == that.numPartitions && + Objects.equals(partitionDimensions, that.partitionDimensions); + } + + @Override + public int hashCode() + { + return Objects.hash(partitionDimensions, numPartitions); + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/ImmutablePartitionHolder.java b/core/src/main/java/org/apache/druid/timeline/partition/ImmutablePartitionHolder.java index 5003f651c121..65c1b0f97baf 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/ImmutablePartitionHolder.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/ImmutablePartitionHolder.java @@ -19,9 +19,11 @@ package org.apache.druid.timeline.partition; +import org.apache.druid.timeline.Overshadowable; + /** */ -public class ImmutablePartitionHolder extends PartitionHolder +public class ImmutablePartitionHolder> extends PartitionHolder { public ImmutablePartitionHolder(PartitionHolder partitionHolder) { diff --git a/core/src/main/java/org/apache/druid/timeline/partition/IntegerPartitionChunk.java b/core/src/main/java/org/apache/druid/timeline/partition/IntegerPartitionChunk.java index 986f8fe2a0d2..8855f0151e8c 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/IntegerPartitionChunk.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/IntegerPartitionChunk.java @@ -19,16 +19,18 @@ package org.apache.druid.timeline.partition; +import org.apache.druid.timeline.Overshadowable; + /** */ -public class IntegerPartitionChunk implements PartitionChunk +public class IntegerPartitionChunk implements PartitionChunk { private final Integer start; private final Integer end; private final int chunkNumber; private final T object; - public static IntegerPartitionChunk make(Integer start, Integer end, int chunkNumber, T obj) + public static IntegerPartitionChunk make(Integer start, Integer end, int chunkNumber, T obj) { return new IntegerPartitionChunk(start, end, chunkNumber, obj); } diff --git a/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpec.java index d0e60c2c5175..1ebb24e16038 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpec.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpec.java @@ -67,6 +67,12 @@ public boolean possibleInDomain(Map> domain) return true; } + @Override + public boolean isCompatible(Class other) + { + return other == LinearShardSpec.class; + } + @Override public PartitionChunk createChunk(T obj) { diff --git a/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpecFactory.java b/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpecFactory.java new file mode 100644 index 000000000000..b6340ec252ff --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpecFactory.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.annotation.Nullable; + +public class LinearShardSpecFactory implements ShardSpecFactory +{ + private static final LinearShardSpecFactory INSTANCE = new LinearShardSpecFactory(); + + public static LinearShardSpecFactory instance() + { + return INSTANCE; + } + + private LinearShardSpecFactory() + { + } + + @Override + public ShardSpec create(ObjectMapper objectMapper, @Nullable ShardSpec specOfPreviousMaxPartitionId) + { + return new LinearShardSpec( + specOfPreviousMaxPartitionId == null ? 0 : specOfPreviousMaxPartitionId.getPartitionNum() + 1 + ); + } + + @Override + public ShardSpec create(ObjectMapper objectMapper, int partitionId) + { + return new LinearShardSpec(partitionId); + } + + @Override + public Class getShardSpecClass() + { + return LinearShardSpec.class; + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NoneShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/NoneShardSpec.java index 7fb2aab3a7a0..dde92167362a 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/NoneShardSpec.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/NoneShardSpec.java @@ -49,7 +49,7 @@ private NoneShardSpec() @Override public PartitionChunk createChunk(T obj) { - return new SingleElementPartitionChunk(obj); + return new SingleElementPartitionChunk<>(obj); } @Override @@ -83,6 +83,12 @@ public boolean possibleInDomain(Map> domain) return true; } + @Override + public boolean isCompatible(Class other) + { + return other == NoneShardSpec.class; + } + @Override public boolean equals(Object obj) { diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpec.java new file mode 100644 index 000000000000..d5b36576a654 --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpec.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; +import org.apache.druid.data.input.InputRow; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * ShardSpec for segments which overshadow others with their minorVersion. + */ +public class NumberedOverwriteShardSpec implements OverwriteShardSpec +{ + private final int partitionId; + + private final short startRootPartitionId; + private final short endRootPartitionId; // exclusive + private final short minorVersion; + private final short atomicUpdateGroupSize; // number of segments in atomicUpdateGroup + + @JsonCreator + public NumberedOverwriteShardSpec( + @JsonProperty("partitionId") int partitionId, + @JsonProperty("startRootPartitionId") int startRootPartitionId, + @JsonProperty("endRootPartitionId") int endRootPartitionId, + @JsonProperty("minorVersion") short minorVersion, + @JsonProperty("atomicUpdateGroupSize") short atomicUpdateGroupSize + ) + { + Preconditions.checkArgument( + partitionId >= PartitionIds.NON_ROOT_GEN_START_PARTITION_ID + && partitionId < PartitionIds.NON_ROOT_GEN_END_PARTITION_ID, + "partitionNum[%s] >= %s && partitionNum[%s] < %s", + partitionId, + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID, + partitionId, + PartitionIds.NON_ROOT_GEN_END_PARTITION_ID + ); + Preconditions.checkArgument( + startRootPartitionId >= PartitionIds.ROOT_GEN_START_PARTITION_ID + && startRootPartitionId < PartitionIds.ROOT_GEN_END_PARTITION_ID, + "startRootPartitionId[%s] >= %s && startRootPartitionId[%s] < %s", + startRootPartitionId, + PartitionIds.ROOT_GEN_START_PARTITION_ID, + startRootPartitionId, + PartitionIds.ROOT_GEN_END_PARTITION_ID + ); + Preconditions.checkArgument( + endRootPartitionId >= PartitionIds.ROOT_GEN_START_PARTITION_ID + && endRootPartitionId < PartitionIds.ROOT_GEN_END_PARTITION_ID, + "endRootPartitionId[%s] >= %s && endRootPartitionId[%s] < %s", + endRootPartitionId, + PartitionIds.ROOT_GEN_START_PARTITION_ID, + endRootPartitionId, + PartitionIds.ROOT_GEN_END_PARTITION_ID + ); + Preconditions.checkArgument(minorVersion > 0, "minorVersion[%s] > 0", minorVersion); + Preconditions.checkArgument( + atomicUpdateGroupSize > 0 || atomicUpdateGroupSize == PartitionIds.UNKNOWN_ATOMIC_UPDATE_GROUP_SIZE, + "atomicUpdateGroupSize[%s] > 0 or == %s", + atomicUpdateGroupSize, + PartitionIds.UNKNOWN_ATOMIC_UPDATE_GROUP_SIZE + ); + + this.partitionId = partitionId; + this.startRootPartitionId = (short) startRootPartitionId; + this.endRootPartitionId = (short) endRootPartitionId; + this.minorVersion = minorVersion; + this.atomicUpdateGroupSize = atomicUpdateGroupSize; + } + + public NumberedOverwriteShardSpec( + int partitionId, + int startRootPartitionId, + int endRootPartitionId, + short minorVersion + ) + { + this( + partitionId, + startRootPartitionId, + endRootPartitionId, + minorVersion, + PartitionIds.UNKNOWN_ATOMIC_UPDATE_GROUP_SIZE + ); + } + + @Override + public OverwriteShardSpec withAtomicUpdateGroupSize(short atomicUpdateGroupSize) + { + return new NumberedOverwriteShardSpec( + this.partitionId, + this.startRootPartitionId, + this.endRootPartitionId, + this.minorVersion, + atomicUpdateGroupSize + ); + } + + @Override + public PartitionChunk createChunk(T obj) + { + return new NumberedOverwritingPartitionChunk<>(partitionId, obj); + } + + @Override + public boolean isInChunk(long timestamp, InputRow inputRow) + { + return true; + } + + @JsonProperty("partitionId") + @Override + public int getPartitionNum() + { + return partitionId; + } + + @JsonProperty + @Override + public int getStartRootPartitionId() + { + return Short.toUnsignedInt(startRootPartitionId); + } + + @JsonProperty + @Override + public int getEndRootPartitionId() + { + return Short.toUnsignedInt(endRootPartitionId); + } + + @JsonProperty + @Override + public short getMinorVersion() + { + return minorVersion; + } + + @JsonProperty + @Override + public short getAtomicUpdateGroupSize() + { + return atomicUpdateGroupSize; + } + + @Override + public ShardSpecLookup getLookup(List shardSpecs) + { + return (long timestamp, InputRow row) -> shardSpecs.get(0); + } + + @Override + public List getDomainDimensions() + { + return Collections.emptyList(); + } + + @Override + public boolean possibleInDomain(Map> domain) + { + return true; + } + + @Override + public boolean isCompatible(Class other) + { + return other == NumberedOverwriteShardSpec.class || other == NumberedShardSpec.class; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + NumberedOverwriteShardSpec that = (NumberedOverwriteShardSpec) o; + return partitionId == that.partitionId && + startRootPartitionId == that.startRootPartitionId && + endRootPartitionId == that.endRootPartitionId && + minorVersion == that.minorVersion && + atomicUpdateGroupSize == that.atomicUpdateGroupSize; + } + + @Override + public int hashCode() + { + return Objects.hash(partitionId, startRootPartitionId, endRootPartitionId, minorVersion, atomicUpdateGroupSize); + } + + @Override + public String toString() + { + return "NumberedOverwriteShardSpec{" + + "partitionId=" + partitionId + + ", startRootPartitionId=" + startRootPartitionId + + ", endRootPartitionId=" + endRootPartitionId + + ", minorVersion=" + minorVersion + + ", atomicUpdateGroupSize=" + atomicUpdateGroupSize + + '}'; + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwritingPartitionChunk.java b/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwritingPartitionChunk.java new file mode 100644 index 000000000000..3c6c9829059a --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwritingPartitionChunk.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.IAE; + +import java.util.Objects; + +/** + * PartitionChunk corresponding to {@link NumberedOverwriteShardSpec} + */ +public class NumberedOverwritingPartitionChunk implements PartitionChunk +{ + private final int chunkId; + private final T object; + + public NumberedOverwritingPartitionChunk(int chunkId, T object) + { + Preconditions.checkArgument( + chunkId >= PartitionIds.NON_ROOT_GEN_START_PARTITION_ID && chunkId < PartitionIds.NON_ROOT_GEN_END_PARTITION_ID, + "partitionNum[%s] >= %s && partitionNum[%s] < %s", + chunkId, + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID, + chunkId, + PartitionIds.NON_ROOT_GEN_END_PARTITION_ID + ); + + this.chunkId = chunkId; + this.object = object; + } + + @Override + public T getObject() + { + return object; + } + + @Override + public boolean abuts(PartitionChunk other) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isStart() + { + return true; + } + + @Override + public boolean isEnd() + { + return true; + } + + @Override + public int getChunkNumber() + { + return chunkId; + } + + @Override + public int compareTo(PartitionChunk o) + { + if (o instanceof NumberedOverwritingPartitionChunk) { + final NumberedOverwritingPartitionChunk that = (NumberedOverwritingPartitionChunk) o; + return Integer.compare(chunkId, that.chunkId); + } else { + throw new IAE("Cannot compare against [%s]", o.getClass().getName()); + } + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + NumberedOverwritingPartitionChunk that = (NumberedOverwritingPartitionChunk) o; + return chunkId == that.chunkId; + } + + @Override + public int hashCode() + { + return Objects.hash(chunkId); + } + + @Override + public String toString() + { + return "NumberedOverwritingPartitionChunk{" + + "chunkId=" + chunkId + + ", object=" + object + + '}'; + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwritingShardSpecFactory.java b/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwritingShardSpecFactory.java new file mode 100644 index 000000000000..764917c9346b --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwritingShardSpecFactory.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.annotation.Nullable; + +public class NumberedOverwritingShardSpecFactory implements ShardSpecFactory +{ + private final int startRootPartitionId; + private final int endRootPartitionId; + private final short minorVersion; + + @JsonCreator + public NumberedOverwritingShardSpecFactory( + @JsonProperty("startRootPartitionId") int startRootPartitionId, + @JsonProperty("endRootPartitionId") int endRootPartitionId, + @JsonProperty("minorVersion") short minorVersion + ) + { + this.startRootPartitionId = startRootPartitionId; + this.endRootPartitionId = endRootPartitionId; + this.minorVersion = minorVersion; + } + + @JsonProperty + public int getStartRootPartitionId() + { + return startRootPartitionId; + } + + @JsonProperty + public int getEndRootPartitionId() + { + return endRootPartitionId; + } + + @JsonProperty + public short getMinorVersion() + { + return minorVersion; + } + + @Override + public ShardSpec create(ObjectMapper objectMapper, @Nullable ShardSpec specOfPreviousMaxPartitionId) + { + // specOfPreviousMaxPartitionId is the max partitionId of the same shardSpec + // and could be null if all existing segments are first-generation segments. + return new NumberedOverwriteShardSpec( + specOfPreviousMaxPartitionId == null + ? PartitionIds.NON_ROOT_GEN_START_PARTITION_ID + : specOfPreviousMaxPartitionId.getPartitionNum() + 1, + startRootPartitionId, + endRootPartitionId, + minorVersion + ); + } + + @Override + public ShardSpec create(ObjectMapper objectMapper, int partitionId) + { + return new NumberedOverwriteShardSpec(partitionId, startRootPartitionId, endRootPartitionId, minorVersion); + } + + @Override + public Class getShardSpecClass() + { + return NumberedOverwriteShardSpec.class; + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NumberedPartitionChunk.java b/core/src/main/java/org/apache/druid/timeline/partition/NumberedPartitionChunk.java index 5ab89f6e8084..2d32532bece0 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/NumberedPartitionChunk.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/NumberedPartitionChunk.java @@ -123,4 +123,14 @@ public int hashCode() { return Objects.hashCode(chunks, chunkNumber); } + + @Override + public String toString() + { + return "NumberedPartitionChunk{" + + "chunkNumber=" + chunkNumber + + ", chunks=" + chunks + + ", object=" + object + + '}'; + } } diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpec.java index 8b555e26fe80..d6f98d71751d 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpec.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpec.java @@ -83,6 +83,12 @@ public boolean possibleInDomain(Map> domain) return true; } + @Override + public boolean isCompatible(Class other) + { + return other == NumberedShardSpec.class || other == NumberedOverwriteShardSpec.class; + } + @JsonProperty("partitions") public int getPartitions() { diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpecFactory.java b/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpecFactory.java new file mode 100644 index 000000000000..486f9edcaa00 --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpecFactory.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.annotation.Nullable; + +public class NumberedShardSpecFactory implements ShardSpecFactory +{ + private static final NumberedShardSpecFactory INSTANCE = new NumberedShardSpecFactory(); + + public static NumberedShardSpecFactory instance() + { + return INSTANCE; + } + + private NumberedShardSpecFactory() + { + } + + @Override + public ShardSpec create(ObjectMapper objectMapper, @Nullable ShardSpec specOfPreviousMaxPartitionId) + { + if (specOfPreviousMaxPartitionId == null) { + return new NumberedShardSpec(0, 0); + } else { + final NumberedShardSpec prevSpec = (NumberedShardSpec) specOfPreviousMaxPartitionId; + return new NumberedShardSpec(prevSpec.getPartitionNum() + 1, prevSpec.getPartitions()); + } + } + + @Override + public ShardSpec create(ObjectMapper objectMapper, int partitionId) + { + return new NumberedShardSpec(partitionId, 0); + } + + @Override + public Class getShardSpecClass() + { + return NumberedShardSpec.class; + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/OvershadowableManager.java b/core/src/main/java/org/apache/druid/timeline/partition/OvershadowableManager.java new file mode 100644 index 000000000000..97e8b9aae92f --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/OvershadowableManager.java @@ -0,0 +1,805 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import it.unimi.dsi.fastutil.objects.AbstractObjectCollection; +import it.unimi.dsi.fastutil.objects.ObjectCollection; +import it.unimi.dsi.fastutil.objects.ObjectIterator; +import it.unimi.dsi.fastutil.objects.ObjectIterators; +import it.unimi.dsi.fastutil.objects.ObjectSortedSet; +import it.unimi.dsi.fastutil.objects.ObjectSortedSets; +import it.unimi.dsi.fastutil.shorts.AbstractShort2ObjectSortedMap; +import it.unimi.dsi.fastutil.shorts.Short2ObjectMap; +import it.unimi.dsi.fastutil.shorts.Short2ObjectRBTreeMap; +import it.unimi.dsi.fastutil.shorts.Short2ObjectSortedMap; +import it.unimi.dsi.fastutil.shorts.ShortComparator; +import it.unimi.dsi.fastutil.shorts.ShortComparators; +import it.unimi.dsi.fastutil.shorts.ShortSortedSet; +import it.unimi.dsi.fastutil.shorts.ShortSortedSets; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.timeline.Overshadowable; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.TreeMap; + +/** + * OvershadowableManager manages the state of {@link AtomicUpdateGroup}. See the below {@link State} for details of + * the possible state. + * Note that an AtomicUpdateGroup can consist of {@link Overshadowable}s of the same majorVersion, minorVersion, + * rootPartition range, and atomicUpdateGroupSize. + * In {@link org.apache.druid.timeline.VersionedIntervalTimeline}, this class is used to manage segments in the same + * timeChunk. + * + * This class is not thread-safe. + */ +class OvershadowableManager> +{ + private enum State + { + STANDBY, // have atomicUpdateGroup of higher versions than visible + VISIBLE, // have a single fully available atomicUpdateGroup of highest version + OVERSHADOWED // have atomicUpdateGroup of lower versions than visible + } + + private final Map> knownPartitionChunks; // served segments + + // (start partitionId, end partitionId) -> minorVersion -> atomicUpdateGroup + private final TreeMap>> standbyGroups; + private final TreeMap>> visibleGroup; + private final TreeMap>> overshadowedGroups; + + OvershadowableManager() + { + this.knownPartitionChunks = new HashMap<>(); + this.standbyGroups = new TreeMap<>(); + this.visibleGroup = new TreeMap<>(); + this.overshadowedGroups = new TreeMap<>(); + } + + OvershadowableManager(OvershadowableManager other) + { + this.knownPartitionChunks = new HashMap<>(other.knownPartitionChunks); + this.standbyGroups = new TreeMap<>(other.standbyGroups); + this.visibleGroup = new TreeMap<>(other.visibleGroup); + this.overshadowedGroups = new TreeMap<>(other.overshadowedGroups); + } + + private TreeMap>> getStateMap(State state) + { + switch (state) { + case STANDBY: + return standbyGroups; + case VISIBLE: + return visibleGroup; + case OVERSHADOWED: + return overshadowedGroups; + default: + throw new ISE("Unknown state[%s]", state); + } + } + + private Short2ObjectSortedMap> createMinorVersionToAugMap(State state) + { + switch (state) { + case STANDBY: + case OVERSHADOWED: + return new Short2ObjectRBTreeMap<>(); + case VISIBLE: + return new SingleEntryShort2ObjectSortedMap<>(); + default: + throw new ISE("Unknown state[%s]", state); + } + } + + private void transitAtomicUpdateGroupState(AtomicUpdateGroup atomicUpdateGroup, State from, State to) + { + Preconditions.checkNotNull(atomicUpdateGroup, "atomicUpdateGroup"); + Preconditions.checkArgument(!atomicUpdateGroup.isEmpty(), "empty atomicUpdateGroup"); + + removeFrom(atomicUpdateGroup, from); + addAtomicUpdateGroupWithState(atomicUpdateGroup, to); + } + + /** + * Find the {@link AtomicUpdateGroup} of the given state which has the same {@link RootPartitionRange} and + * minorVersion with {@link PartitionChunk}. + */ + @Nullable + private AtomicUpdateGroup findAtomicUpdateGroupWith(PartitionChunk chunk, State state) + { + final Short2ObjectSortedMap> versionToGroup = getStateMap(state).get( + RootPartitionRange.of(chunk) + ); + if (versionToGroup != null) { + final AtomicUpdateGroup atomicUpdateGroup = versionToGroup.get(chunk.getObject().getMinorVersion()); + if (atomicUpdateGroup != null) { + return atomicUpdateGroup; + } + } + return null; + } + + /** + * Returns null if atomicUpdateGroup is not found for the state. + * Can return an empty atomicUpdateGroup. + */ + @Nullable + private AtomicUpdateGroup tryRemoveChunkFromGroupWithState(PartitionChunk chunk, State state) + { + final RootPartitionRange rangeKey = RootPartitionRange.of(chunk); + final Short2ObjectSortedMap> versionToGroup = getStateMap(state).get(rangeKey); + if (versionToGroup != null) { + final AtomicUpdateGroup atomicUpdateGroup = versionToGroup.get(chunk.getObject().getMinorVersion()); + if (atomicUpdateGroup != null) { + atomicUpdateGroup.remove(chunk); + if (atomicUpdateGroup.isEmpty()) { + versionToGroup.remove(chunk.getObject().getMinorVersion()); + if (versionToGroup.isEmpty()) { + getStateMap(state).remove(rangeKey); + } + } + + determineVisibleGroupAfterRemove( + atomicUpdateGroup, + RootPartitionRange.of(chunk), + chunk.getObject().getMinorVersion(), + state + ); + return atomicUpdateGroup; + } + } + return null; + } + + private List>> findOvershadowedBy( + AtomicUpdateGroup aug, + State fromState + ) + { + final RootPartitionRange rangeKeyOfGivenAug = RootPartitionRange.of(aug); + return findOvershadowedBy(rangeKeyOfGivenAug, aug.getMinorVersion(), fromState); + } + + /** + * Find all atomicUpdateGroups of the given state overshadowed by the given rootPartitionRange and minorVersion. + * The atomicUpdateGroup of a higher minorVersion can have a wider RootPartitionRange. + * To find all atomicUpdateGroups overshadowed by the given rootPartitionRange and minorVersion, + * we first need to find the first key contained by the given rootPartitionRange. + * Once we find such key, then we go through the entire map until we see an atomicUpdateGroup of which + * rootRangePartition is not contained by the given rootPartitionRange. + */ + private List>> findOvershadowedBy( + RootPartitionRange rangeOfAug, + short minorVersion, + State fromState + ) + { + final TreeMap>> stateMap = getStateMap(fromState); + Entry>> current = stateMap.floorEntry(rangeOfAug); + + if (current == null) { + return Collections.emptyList(); + } + + // Find the first key for searching for overshadowed atomicUpdateGroup + while (true) { + final Entry>> lowerEntry = stateMap.lowerEntry( + current.getKey() + ); + if (lowerEntry != null && lowerEntry.getKey().startPartitionId == rangeOfAug.startPartitionId) { + current = lowerEntry; + } else { + break; + } + } + + // Going through the map to find all entries of the RootPartitionRange contained by the given rangeOfAug. + // Note that RootPartitionRange of entries are always consecutive. + final List>> found = new ArrayList<>(); + while (current != null && rangeOfAug.contains(current.getKey())) { + // versionToGroup is sorted by minorVersion. + // versionToGroup.subMap(firstKey, minorVersion) below returns a map containing all entries of lower minorVersions + // than the given minorVersion. + final Short2ObjectSortedMap> versionToGroup = current.getValue(); + // Short2ObjectRBTreeMap.SubMap.short2ObjectEntrySet() implementation, especially size(), is not optimized. + // Note that size() is indirectly called in ArrayList.addAll() when ObjectSortedSet.toArray() is called. + // See AbstractObjectCollection.toArray(). + // If you see performance degradation here, probably we need to improve the below line. + found.addAll(versionToGroup.subMap(versionToGroup.firstShortKey(), minorVersion).short2ObjectEntrySet()); + current = stateMap.higherEntry(current.getKey()); + } + return found; + } + + /** + * Handles addition of the atomicUpdateGroup to the given state + */ + private void transitionStandbyGroupIfFull(AtomicUpdateGroup aug, State stateOfAug) + { + if (stateOfAug == State.STANDBY) { + // A standby atomicUpdateGroup becomes visible when its all segments are available. + if (aug.isFull()) { + // A visible atomicUpdateGroup becomes overshadowed when a fully available standby atomicUpdateGroup becomes + // visible which overshadows the current visible one. + findOvershadowedBy(aug, State.VISIBLE) + .forEach(entry -> transitAtomicUpdateGroupState(entry.getValue(), State.VISIBLE, State.OVERSHADOWED)); + findOvershadowedBy(aug, State.STANDBY) + .forEach(entry -> transitAtomicUpdateGroupState(entry.getValue(), State.STANDBY, State.OVERSHADOWED)); + transitAtomicUpdateGroupState(aug, State.STANDBY, State.VISIBLE); + } + } + } + + private void addAtomicUpdateGroupWithState(AtomicUpdateGroup aug, State state) + { + final AtomicUpdateGroup existing = getStateMap(state) + .computeIfAbsent(RootPartitionRange.of(aug), k -> createMinorVersionToAugMap(state)) + .put(aug.getMinorVersion(), aug); + + if (existing != null) { + throw new ISE("AtomicUpdateGroup[%s] is already in state[%s]", aug, state); + } + + transitionStandbyGroupIfFull(aug, state); + } + + public boolean addChunk(PartitionChunk chunk) + { + // Sanity check. ExistingChunk should be usually null. + final PartitionChunk existingChunk = knownPartitionChunks.put(chunk.getChunkNumber(), chunk); + if (existingChunk != null) { + if (!existingChunk.equals(chunk)) { + throw new ISE( + "existingChunk[%s] is different from newChunk[%s] for partitionId[%d]", + existingChunk, + chunk, + chunk.getChunkNumber() + ); + } else { + // A new chunk of the same major version and partitionId can be added in segment handoff + // from stream ingestion tasks to historicals + return false; + } + } + + // Find atomicUpdateGroup of the new chunk + AtomicUpdateGroup atomicUpdateGroup = findAtomicUpdateGroupWith(chunk, State.OVERSHADOWED); + + if (atomicUpdateGroup != null) { + atomicUpdateGroup.add(chunk); + } else { + atomicUpdateGroup = findAtomicUpdateGroupWith(chunk, State.STANDBY); + + if (atomicUpdateGroup != null) { + atomicUpdateGroup.add(chunk); + transitionStandbyGroupIfFull(atomicUpdateGroup, State.STANDBY); + } else { + atomicUpdateGroup = findAtomicUpdateGroupWith(chunk, State.VISIBLE); + + if (atomicUpdateGroup != null) { + atomicUpdateGroup.add(chunk); + } else { + final AtomicUpdateGroup newAtomicUpdateGroup = new AtomicUpdateGroup<>(chunk); + + // Decide the initial state of the new atomicUpdateGroup + final boolean overshadowed = visibleGroup + .values() + .stream() + .flatMap(map -> map.values().stream()) + .anyMatch(group -> group.overshadows(newAtomicUpdateGroup)); + + if (overshadowed) { + addAtomicUpdateGroupWithState(newAtomicUpdateGroup, State.OVERSHADOWED); + } else { + addAtomicUpdateGroupWithState(newAtomicUpdateGroup, State.STANDBY); + } + } + } + } + return true; + } + + /** + * Handles of removal of an empty atomicUpdateGroup from a state. + */ + private void determineVisibleGroupAfterRemove( + AtomicUpdateGroup augOfRemovedChunk, + RootPartitionRange rangeOfAug, + short minorVersion, + State stateOfRemovedAug + ) + { + // If an atomicUpdateGroup is overshadowed by another non-visible atomicUpdateGroup, there must be another visible + // atomicUpdateGroup which also overshadows the same atomicUpdateGroup. + // As a result, the state of overshadowed atomicUpdateGroup should be updated only when a visible atomicUpdateGroup + // is removed. + + if (stateOfRemovedAug == State.VISIBLE) { + // All segments in the visible atomicUpdateGroup which overshadows this atomicUpdateGroup is removed. + // Fall back if there is a fully available overshadowed atomicUpdateGroup + + final List> latestFullAugs = findLatestFullyAvailableOvershadowedAtomicUpdateGroup( + rangeOfAug, + minorVersion + ); + + // If there is no fully available fallback group, then the existing VISIBLE group remains VISIBLE. + // Otherwise, the latest fully available group becomes VISIBLE. + if (!latestFullAugs.isEmpty()) { + // Move the atomicUpdateGroup to standby + // and move the fully available overshadowed atomicUpdateGroup to visible + if (!augOfRemovedChunk.isEmpty()) { + transitAtomicUpdateGroupState(augOfRemovedChunk, State.VISIBLE, State.STANDBY); + } + latestFullAugs.forEach(group -> transitAtomicUpdateGroupState(group, State.OVERSHADOWED, State.VISIBLE)); + } + } + } + + private List> findLatestFullyAvailableOvershadowedAtomicUpdateGroup( + RootPartitionRange rangeOfAug, + short minorVersion + ) + { + final List>> overshadowedGroups = findOvershadowedBy( + rangeOfAug, + minorVersion, + State.OVERSHADOWED + ); + if (overshadowedGroups.isEmpty()) { + return Collections.emptyList(); + } + + final OvershadowableManager manager = new OvershadowableManager<>(); + for (Short2ObjectMap.Entry> entry : overshadowedGroups) { + for (PartitionChunk chunk : entry.getValue().getChunks()) { + manager.addChunk(chunk); + } + } + + final List> visibles = new ArrayList<>(); + for (Short2ObjectSortedMap> map : manager.visibleGroup.values()) { + visibles.addAll(map.values()); + } + return visibles; + } + + private void removeFrom(AtomicUpdateGroup aug, State state) + { + final RootPartitionRange rangeKey = RootPartitionRange.of(aug); + final Short2ObjectSortedMap> versionToGroup = getStateMap(state).get(rangeKey); + if (versionToGroup == null) { + throw new ISE("Unknown atomicUpdateGroup[%s] in state[%s]", aug, state); + } + + final AtomicUpdateGroup removed = versionToGroup.remove(aug.getMinorVersion()); + if (removed == null) { + throw new ISE("Unknown atomicUpdateGroup[%s] in state[%s]", aug, state); + } + + if (!removed.equals(aug)) { + throw new ISE( + "WTH? actually removed atomicUpdateGroup[%s] is different from the one which is supposed to be[%s]", + removed, + aug + ); + } + + if (versionToGroup.isEmpty()) { + getStateMap(state).remove(rangeKey); + } + } + + @Nullable + public PartitionChunk removeChunk(PartitionChunk partitionChunk) + { + final PartitionChunk knownChunk = knownPartitionChunks.get(partitionChunk.getChunkNumber()); + if (knownChunk == null) { + return null; + } + + if (!knownChunk.equals(partitionChunk)) { + throw new ISE( + "WTH? Same partitionId[%d], but known partition[%s] is different from the input partition[%s]", + partitionChunk.getChunkNumber(), + knownChunk, + partitionChunk + ); + } + + AtomicUpdateGroup augOfRemovedChunk = tryRemoveChunkFromGroupWithState(partitionChunk, State.STANDBY); + + if (augOfRemovedChunk == null) { + augOfRemovedChunk = tryRemoveChunkFromGroupWithState(partitionChunk, State.VISIBLE); + if (augOfRemovedChunk == null) { + augOfRemovedChunk = tryRemoveChunkFromGroupWithState(partitionChunk, State.OVERSHADOWED); + if (augOfRemovedChunk == null) { + throw new ISE("Can't find atomicUpdateGroup for partitionChunk[%s]", partitionChunk); + } + } + } + + return knownPartitionChunks.remove(partitionChunk.getChunkNumber()); + } + + public boolean isEmpty() + { + return visibleGroup.isEmpty(); + } + + public boolean isComplete() + { + return visibleGroup.values().stream().allMatch(map -> Iterables.getOnlyElement(map.values()).isFull()); + } + + @Nullable + public PartitionChunk getChunk(int partitionId) + { + final PartitionChunk chunk = knownPartitionChunks.get(partitionId); + if (chunk == null) { + return null; + } + final AtomicUpdateGroup aug = findAtomicUpdateGroupWith(chunk, State.VISIBLE); + if (aug == null) { + return null; + } else { + return Preconditions.checkNotNull( + aug.findChunk(partitionId), + "Can't find partitionChunk for partitionId[%s] in atomicUpdateGroup[%s]", + partitionId, + aug + ); + } + } + + public List> getVisibles() + { + final List> visibles = new ArrayList<>(); + for (Short2ObjectSortedMap> treeMap : visibleGroup.values()) { + for (AtomicUpdateGroup aug : treeMap.values()) { + visibles.addAll(aug.getChunks()); + } + } + return visibles; + } + + public List> getOvershadowed() + { + final List> overshadowed = new ArrayList<>(); + for (Short2ObjectSortedMap> treeMap : overshadowedGroups.values()) { + for (AtomicUpdateGroup aug : treeMap.values()) { + overshadowed.addAll(aug.getChunks()); + } + } + return overshadowed; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OvershadowableManager that = (OvershadowableManager) o; + return Objects.equals(knownPartitionChunks, that.knownPartitionChunks) && + Objects.equals(standbyGroups, that.standbyGroups) && + Objects.equals(visibleGroup, that.visibleGroup) && + Objects.equals(overshadowedGroups, that.overshadowedGroups); + } + + @Override + public int hashCode() + { + return Objects.hash(knownPartitionChunks, standbyGroups, visibleGroup, overshadowedGroups); + } + + @Override + public String toString() + { + return "OvershadowableManager{" + + "knownPartitionChunks=" + knownPartitionChunks + + ", standbyGroups=" + standbyGroups + + ", visibleGroup=" + visibleGroup + + ", overshadowedGroups=" + overshadowedGroups + + '}'; + } + + private static class RootPartitionRange implements Comparable + { + private final short startPartitionId; + private final short endPartitionId; + + private static > RootPartitionRange of(PartitionChunk chunk) + { + return of(chunk.getObject().getStartRootPartitionId(), chunk.getObject().getEndRootPartitionId()); + } + + private static > RootPartitionRange of(AtomicUpdateGroup aug) + { + return of(aug.getStartRootPartitionId(), aug.getEndRootPartitionId()); + } + + private static RootPartitionRange of(int startPartitionId, int endPartitionId) + { + return new RootPartitionRange((short) startPartitionId, (short) endPartitionId); + } + + private RootPartitionRange(short startPartitionId, short endPartitionId) + { + this.startPartitionId = startPartitionId; + this.endPartitionId = endPartitionId; + } + + public boolean contains(RootPartitionRange that) + { + return this.startPartitionId <= that.startPartitionId && this.endPartitionId >= that.endPartitionId; + } + + @Override + public int compareTo(RootPartitionRange o) + { + if (startPartitionId != o.startPartitionId) { + return Integer.compare(Short.toUnsignedInt(startPartitionId), Short.toUnsignedInt(o.startPartitionId)); + } else { + return Integer.compare(Short.toUnsignedInt(endPartitionId), Short.toUnsignedInt(o.endPartitionId)); + } + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RootPartitionRange that = (RootPartitionRange) o; + return startPartitionId == that.startPartitionId && + endPartitionId == that.endPartitionId; + } + + @Override + public int hashCode() + { + return Objects.hash(startPartitionId, endPartitionId); + } + + @Override + public String toString() + { + return "RootPartitionRange{" + + "startPartitionId=" + startPartitionId + + ", endPartitionId=" + endPartitionId + + '}'; + } + } + + /** + * Map can store at most a single entry. + * Comparing to{@link it.unimi.dsi.fastutil.shorts.Short2ObjectSortedMaps.Singleton}, it's different from the + * perspective of that this class supports update. + */ + private static class SingleEntryShort2ObjectSortedMap extends AbstractShort2ObjectSortedMap + { + private short key; + private V val; + + private SingleEntryShort2ObjectSortedMap() + { + key = -1; + val = null; + } + + @Override + public Short2ObjectSortedMap subMap(short fromKey, short toKey) + { + if (fromKey <= key && toKey > key) { + return this; + } else { + throw new IAE("fromKey: %s, toKey: %s, key: %s", fromKey, toKey, key); + } + } + + @Override + public Short2ObjectSortedMap headMap(short toKey) + { + if (toKey > key) { + return this; + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public Short2ObjectSortedMap tailMap(short fromKey) + { + if (fromKey <= key) { + return this; + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public short firstShortKey() + { + if (key < 0) { + throw new NoSuchElementException(); + } + return key; + } + + @Override + public short lastShortKey() + { + if (key < 0) { + throw new NoSuchElementException(); + } + return key; + } + + @Override + public ObjectSortedSet> short2ObjectEntrySet() + { + return isEmpty() ? ObjectSortedSets.EMPTY_SET : ObjectSortedSets.singleton(new BasicEntry<>(key, val)); + } + + @Override + public ShortSortedSet keySet() + { + return isEmpty() ? ShortSortedSets.EMPTY_SET : ShortSortedSets.singleton(key); + } + + @Override + public ObjectCollection values() + { + return new AbstractObjectCollection() + { + @Override + public ObjectIterator iterator() + { + return size() > 0 ? ObjectIterators.singleton(val) : ObjectIterators.emptyIterator(); + } + + @Override + public int size() + { + return key < 0 ? 0 : 1; + } + }; + } + + @Override + public V put(final short key, final V value) + { + if (isEmpty()) { + this.key = key; + this.val = value; + return null; + } else { + if (this.key == key) { + final V existing = this.val; + this.val = value; + return existing; + } else { + throw new ISE( + "Can't add [%d, %s] to non-empty SingleEntryShort2ObjectSortedMap[%d, %s]", + key, + value, + this.key, + this.val + ); + } + } + } + + @Override + public V get(short key) + { + return this.key == key ? val : null; + } + + @Override + public V remove(final short key) + { + if (this.key == key) { + this.key = -1; + return val; + } else { + return null; + } + } + + @Override + public boolean containsKey(short key) + { + return this.key == key; + } + + @Override + public ShortComparator comparator() + { + return ShortComparators.NATURAL_COMPARATOR; + } + + @Override + public int size() + { + return key < 0 ? 0 : 1; + } + + @Override + public void defaultReturnValue(V rv) + { + throw new UnsupportedOperationException(); + } + + @Override + public V defaultReturnValue() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isEmpty() + { + return key < 0; + } + + @Override + public boolean containsValue(Object value) + { + if (key < 0) { + return false; + } else { + return Objects.equals(val, value); + } + } + + @Override + public void putAll(Map m) + { + if (!m.isEmpty()) { + if (m.size() == 1) { + final Map.Entry entry = m.entrySet().iterator().next(); + this.key = entry.getKey(); + this.val = entry.getValue(); + } else { + throw new IllegalArgumentException(); + } + } + } + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/OverwriteShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/OverwriteShardSpec.java new file mode 100644 index 000000000000..0fea5646e311 --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/OverwriteShardSpec.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +/** + * ShardSpec for non-first-generation segments. + * This shardSpec is allocated a partitionId between {@link PartitionIds#NON_ROOT_GEN_START_PARTITION_ID} and + * {@link PartitionIds#NON_ROOT_GEN_END_PARTITION_ID}. + * + * @see org.apache.druid.timeline.Overshadowable + */ +public interface OverwriteShardSpec extends ShardSpec +{ + default OverwriteShardSpec withAtomicUpdateGroupSize(int atomicUpdateGroupSize) + { + return withAtomicUpdateGroupSize((short) atomicUpdateGroupSize); + } + + OverwriteShardSpec withAtomicUpdateGroupSize(short atomicUpdateGroupSize); +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/PartitionHolder.java b/core/src/main/java/org/apache/druid/timeline/partition/PartitionHolder.java index dcf29aedc488..26e34dacf76d 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/PartitionHolder.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/PartitionHolder.java @@ -20,30 +20,32 @@ package org.apache.druid.timeline.partition; import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; +import org.apache.druid.timeline.Overshadowable; import javax.annotation.Nullable; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Spliterator; -import java.util.TreeMap; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; /** * An object that clumps together multiple other objects which each represent a shard of some space. */ -public class PartitionHolder implements Iterable> +public class PartitionHolder> implements Iterable> { - private final TreeMap, PartitionChunk> holderMap; + private final OvershadowableManager overshadowableManager; public PartitionHolder(PartitionChunk initialChunk) { - this.holderMap = new TreeMap<>(); + this.overshadowableManager = new OvershadowableManager<>(); add(initialChunk); } public PartitionHolder(List> initialChunks) { - this.holderMap = new TreeMap<>(); + this.overshadowableManager = new OvershadowableManager<>(); for (PartitionChunk chunk : initialChunks) { add(chunk); } @@ -51,33 +53,32 @@ public PartitionHolder(List> initialChunks) public PartitionHolder(PartitionHolder partitionHolder) { - this.holderMap = new TreeMap<>(); - this.holderMap.putAll(partitionHolder.holderMap); + this.overshadowableManager = new OvershadowableManager<>(partitionHolder.overshadowableManager); } public boolean add(PartitionChunk chunk) { - return holderMap.putIfAbsent(chunk, chunk) == null; + return overshadowableManager.addChunk(chunk); } @Nullable public PartitionChunk remove(PartitionChunk chunk) { - return holderMap.remove(chunk); + return overshadowableManager.removeChunk(chunk); } public boolean isEmpty() { - return holderMap.isEmpty(); + return overshadowableManager.isEmpty(); } public boolean isComplete() { - if (holderMap.isEmpty()) { + if (overshadowableManager.isEmpty()) { return false; } - Iterator> iter = holderMap.keySet().iterator(); + Iterator> iter = iterator(); PartitionChunk curr = iter.next(); @@ -86,7 +87,7 @@ public boolean isComplete() } if (curr.isEnd()) { - return true; + return overshadowableManager.isComplete(); } while (iter.hasNext()) { @@ -96,7 +97,7 @@ public boolean isComplete() } if (next.isEnd()) { - return true; + return overshadowableManager.isComplete(); } curr = next; } @@ -106,24 +107,29 @@ public boolean isComplete() public PartitionChunk getChunk(final int partitionNum) { - final Iterator> retVal = Iterators.filter( - holderMap.keySet().iterator(), - input -> input.getChunkNumber() == partitionNum - ); - - return retVal.hasNext() ? retVal.next() : null; + return overshadowableManager.getChunk(partitionNum); } @Override public Iterator> iterator() { - return holderMap.keySet().iterator(); + return overshadowableManager.getVisibles().iterator(); } @Override public Spliterator> spliterator() { - return holderMap.keySet().spliterator(); + return overshadowableManager.getVisibles().spliterator(); + } + + public Stream> stream() + { + return StreamSupport.stream(spliterator(), false); + } + + public List> getOvershadowed() + { + return overshadowableManager.getOvershadowed(); } public Iterable payloads() @@ -140,27 +146,21 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - - PartitionHolder that = (PartitionHolder) o; - - if (!holderMap.equals(that.holderMap)) { - return false; - } - - return true; + PartitionHolder that = (PartitionHolder) o; + return Objects.equals(overshadowableManager, that.overshadowableManager); } @Override public int hashCode() { - return holderMap.hashCode(); + return Objects.hash(overshadowableManager); } @Override public String toString() { return "PartitionHolder{" + - "holderMap=" + holderMap + + "overshadowableManager=" + overshadowableManager + '}'; } } diff --git a/core/src/main/java/org/apache/druid/timeline/partition/PartitionIds.java b/core/src/main/java/org/apache/druid/timeline/partition/PartitionIds.java new file mode 100644 index 000000000000..fc5d0e981c45 --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/PartitionIds.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +public final class PartitionIds +{ + /** + * Start partitionId available for root generation segments. + */ + public static final int ROOT_GEN_START_PARTITION_ID = 0; + /** + * End partitionId available for root generation segments. + */ + public static final int ROOT_GEN_END_PARTITION_ID = 32768; // exclusive + /** + * Start partitionId available for non-root generation segments. + */ + public static final int NON_ROOT_GEN_START_PARTITION_ID = 32768; + /** + * End partitionId available for non-root generation segments. + */ + public static final int NON_ROOT_GEN_END_PARTITION_ID = 65536; // exclusive + + public static final short UNKNOWN_ATOMIC_UPDATE_GROUP_SIZE = -1; + + private PartitionIds() + { + } +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/ShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/ShardSpec.java index 10a6d6cf50d0..43aaf701db36 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/ShardSpec.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/ShardSpec.java @@ -37,7 +37,8 @@ @JsonSubTypes.Type(name = "single", value = SingleDimensionShardSpec.class), @JsonSubTypes.Type(name = "linear", value = LinearShardSpec.class), @JsonSubTypes.Type(name = "numbered", value = NumberedShardSpec.class), - @JsonSubTypes.Type(name = "hashed", value = HashBasedNumberedShardSpec.class) + @JsonSubTypes.Type(name = "hashed", value = HashBasedNumberedShardSpec.class), + @JsonSubTypes.Type(name = "numbered_overwrite", value = NumberedOverwriteShardSpec.class) }) public interface ShardSpec { @@ -47,6 +48,26 @@ public interface ShardSpec int getPartitionNum(); + default int getStartRootPartitionId() + { + return getPartitionNum(); + } + + default int getEndRootPartitionId() + { + return getPartitionNum() + 1; + } + + default short getMinorVersion() + { + return 0; + } + + default short getAtomicUpdateGroupSize() + { + return 1; + } + ShardSpecLookup getLookup(List shardSpecs); /** @@ -61,4 +82,9 @@ public interface ShardSpec * @return possibility of in domain */ boolean possibleInDomain(Map> domain); + + /** + * Returns true if two segments of this and other shardSpecs can exist in the same timeChunk. + */ + boolean isCompatible(Class other); } diff --git a/core/src/main/java/org/apache/druid/timeline/partition/ShardSpecFactory.java b/core/src/main/java/org/apache/druid/timeline/partition/ShardSpecFactory.java new file mode 100644 index 000000000000..7ab0be836d59 --- /dev/null +++ b/core/src/main/java/org/apache/druid/timeline/partition/ShardSpecFactory.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.timeline.partition; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.annotation.Nullable; + +/** + * Factory to be used to allocate segments remotely in the overlord. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(name = "numbered", value = NumberedShardSpecFactory.class), + @JsonSubTypes.Type(name = "hashed", value = HashBasedNumberedShardSpecFactory.class), + @JsonSubTypes.Type(name = "numbered_overwrite", value = NumberedOverwritingShardSpecFactory.class), +}) +public interface ShardSpecFactory +{ + /** + * Create a new shardSpec based on {@code specOfPreviousMaxPartitionId}. If it's null, it assumes that this is the + * first call for the timeChunk where the new segment is created. + * Note that {@code specOfPreviousMaxPartitionId} can also be null for {@link OverwriteShardSpec} if all segments + * in the timeChunk are first-generation segments. + */ + ShardSpec create(ObjectMapper objectMapper, @Nullable ShardSpec specOfPreviousMaxPartitionId); + + /** + * Create a new shardSpec having the given partitionId. + */ + ShardSpec create(ObjectMapper objectMapper, int partitionId); + + /** + * Return the class of the shardSpec created by this factory. + */ + Class getShardSpecClass(); +} diff --git a/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionShardSpec.java index f2b84413f171..968a1d74cc98 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionShardSpec.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionShardSpec.java @@ -33,9 +33,7 @@ import java.util.Map; /** - * Class uses getters/setters to work around http://jira.codehaus.org/browse/MSHADE-92 - * - * Adjust to JsonCreator and final fields when resolved. + * {@link ShardSpec} for range partitioning based on a single dimension */ public class SingleDimensionShardSpec implements ShardSpec { @@ -46,6 +44,12 @@ public class SingleDimensionShardSpec implements ShardSpec private final String end; private final int partitionNum; + /** + * @param dimension partition dimension + * @param start inclusive start of this range + * @param end exclusive end of this range + * @param partitionNum unique ID for this shard + */ @JsonCreator public SingleDimensionShardSpec( @JsonProperty("dimension") String dimension, @@ -54,6 +58,7 @@ public SingleDimensionShardSpec( @JsonProperty("partitionNum") int partitionNum ) { + Preconditions.checkArgument(partitionNum >= 0, "partitionNum >= 0"); this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.start = start; this.end = end; @@ -131,6 +136,12 @@ public boolean possibleInDomain(Map> domain) return !rangeSet.subRangeSet(getRange()).isEmpty(); } + @Override + public boolean isCompatible(Class other) + { + return other == SingleDimensionShardSpec.class; + } + @Override public PartitionChunk createChunk(T obj) { diff --git a/core/src/main/java/org/apache/druid/timeline/partition/SingleElementPartitionChunk.java b/core/src/main/java/org/apache/druid/timeline/partition/SingleElementPartitionChunk.java index f12073fccb39..2567fe617552 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/SingleElementPartitionChunk.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/SingleElementPartitionChunk.java @@ -93,7 +93,7 @@ public boolean equals(Object o) @Override public int hashCode() { - return element != null ? element.hashCode() : 0; + return 0; } @Override diff --git a/core/src/main/java/org/apache/druid/timeline/partition/StringPartitionChunk.java b/core/src/main/java/org/apache/druid/timeline/partition/StringPartitionChunk.java index ff7171f571d3..28d7505e70a6 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/StringPartitionChunk.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/StringPartitionChunk.java @@ -116,4 +116,15 @@ public int hashCode() result = 31 * result + (object != null ? object.hashCode() : 0); return result; } + + @Override + public String toString() + { + return "StringPartitionChunk{" + + "start='" + start + '\'' + + ", end='" + end + '\'' + + ", chunkNumber=" + chunkNumber + + ", object=" + object + + '}'; + } } diff --git a/core/src/test/java/org/apache/druid/timeline/DataSegmentTest.java b/core/src/test/java/org/apache/druid/timeline/DataSegmentTest.java index 701578541c64..bc3cf902b9ed 100644 --- a/core/src/test/java/org/apache/druid/timeline/DataSegmentTest.java +++ b/core/src/test/java/org/apache/druid/timeline/DataSegmentTest.java @@ -30,6 +30,7 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.timeline.partition.NoneShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpec; import org.apache.druid.timeline.partition.PartitionChunk; import org.apache.druid.timeline.partition.ShardSpec; import org.apache.druid.timeline.partition.ShardSpecLookup; @@ -92,6 +93,12 @@ public boolean possibleInDomain(Map> domain) { return true; } + + @Override + public boolean isCompatible(Class other) + { + return false; + } }; } @@ -117,7 +124,7 @@ public void testV1Serialization() throws Exception loadSpec, Arrays.asList("dim1", "dim2"), Arrays.asList("met1", "met2"), - NoneShardSpec.instance(), + new NumberedShardSpec(3, 0), TEST_VERSION, 1 ); @@ -134,7 +141,7 @@ public void testV1Serialization() throws Exception Assert.assertEquals(loadSpec, objectMap.get("loadSpec")); Assert.assertEquals("dim1,dim2", objectMap.get("dimensions")); Assert.assertEquals("met1,met2", objectMap.get("metrics")); - Assert.assertEquals(ImmutableMap.of("type", "none"), objectMap.get("shardSpec")); + Assert.assertEquals(ImmutableMap.of("type", "numbered", "partitionNum", 3, "partitions", 0), objectMap.get("shardSpec")); Assert.assertEquals(TEST_VERSION, objectMap.get("binaryVersion")); Assert.assertEquals(1, objectMap.get("size")); diff --git a/core/src/test/java/org/apache/druid/timeline/VersionedIntervalTimelineTest.java b/core/src/test/java/org/apache/druid/timeline/VersionedIntervalTimelineTest.java index 3e66bf53de79..6971833678f9 100644 --- a/core/src/test/java/org/apache/druid/timeline/VersionedIntervalTimelineTest.java +++ b/core/src/test/java/org/apache/druid/timeline/VersionedIntervalTimelineTest.java @@ -19,10 +19,10 @@ package org.apache.druid.timeline; -import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import org.apache.druid.java.util.common.DateTimes; @@ -30,8 +30,11 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.timeline.partition.ImmutablePartitionHolder; import org.apache.druid.timeline.partition.IntegerPartitionChunk; +import org.apache.druid.timeline.partition.NumberedOverwritingPartitionChunk; +import org.apache.druid.timeline.partition.NumberedPartitionChunk; import org.apache.druid.timeline.partition.PartitionChunk; import org.apache.druid.timeline.partition.PartitionHolder; +import org.apache.druid.timeline.partition.PartitionIds; import org.apache.druid.timeline.partition.SingleElementPartitionChunk; import org.joda.time.DateTime; import org.joda.time.Days; @@ -45,13 +48,14 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Set; /** */ public class VersionedIntervalTimelineTest { - VersionedIntervalTimeline timeline; + VersionedIntervalTimeline timeline; @Before public void setUp() @@ -70,8 +74,8 @@ public void setUp() add("2011-05-01/2011-05-10", "4", 9); add("2011-10-01/2011-10-02", "1", 1); - add("2011-10-02/2011-10-03", "3", IntegerPartitionChunk.make(null, 10, 0, 20)); - add("2011-10-02/2011-10-03", "3", IntegerPartitionChunk.make(10, null, 1, 21)); + add("2011-10-02/2011-10-03", "3", IntegerPartitionChunk.make(null, 10, 0, new OvershadowableInteger("3", 0, 20))); + add("2011-10-02/2011-10-03", "3", IntegerPartitionChunk.make(10, null, 1, new OvershadowableInteger("3", 1, 21))); add("2011-10-03/2011-10-04", "3", 3); add("2011-10-04/2011-10-05", "4", 4); add("2011-10-05/2011-10-06", "5", 5); @@ -94,8 +98,8 @@ public void testApril() public void testApril2() { Assert.assertEquals( - makeSingle(1), - timeline.remove(Intervals.of("2011-04-01/2011-04-09"), "2", makeSingle(1)) + makeSingle("2", 1), + timeline.remove(Intervals.of("2011-04-01/2011-04-09"), "2", makeSingle("2", 1)) ); assertValues( Arrays.asList( @@ -112,12 +116,12 @@ public void testApril2() public void testApril3() { Assert.assertEquals( - makeSingle(1), - timeline.remove(Intervals.of("2011-04-01/2011-04-09"), "2", makeSingle(1)) + makeSingle("2", 1), + timeline.remove(Intervals.of("2011-04-01/2011-04-09"), "2", makeSingle("2", 1)) ); Assert.assertEquals( - makeSingle(2), - timeline.remove(Intervals.of("2011-04-01/2011-04-03"), "1", makeSingle(2)) + makeSingle("1", 2), + timeline.remove(Intervals.of("2011-04-01/2011-04-03"), "1", makeSingle("1", 2)) ); assertValues( Arrays.asList( @@ -133,8 +137,8 @@ public void testApril3() public void testApril4() { Assert.assertEquals( - makeSingle(1), - timeline.remove(Intervals.of("2011-04-01/2011-04-09"), "2", makeSingle(1)) + makeSingle("2", 1), + timeline.remove(Intervals.of("2011-04-01/2011-04-09"), "2", makeSingle("2", 1)) ); assertValues( Arrays.asList( @@ -168,7 +172,7 @@ public void testMay() @Test public void testMay2() { - Assert.assertNotNull(timeline.remove(Intervals.of("2011-05-01/2011-05-10"), "4", makeSingle(1))); + Assert.assertNotNull(timeline.remove(Intervals.of("2011-05-01/2011-05-10"), "4", makeSingle("4", 9))); assertValues( Arrays.asList( createExpected("2011-05-01/2011-05-03", "2", 7), @@ -183,12 +187,12 @@ public void testMay2() public void testMay3() { Assert.assertEquals( - makeSingle(9), - timeline.remove(Intervals.of("2011-05-01/2011-05-10"), "4", makeSingle(9)) + makeSingle("4", 9), + timeline.remove(Intervals.of("2011-05-01/2011-05-10"), "4", makeSingle("4", 9)) ); Assert.assertEquals( - makeSingle(7), - timeline.remove(Intervals.of("2011-05-01/2011-05-05"), "2", makeSingle(7)) + makeSingle("2", 7), + timeline.remove(Intervals.of("2011-05-01/2011-05-05"), "2", makeSingle("2", 7)) ); assertValues( Arrays.asList( @@ -227,35 +231,35 @@ public void testInsertInWrongOrder() @Test public void testRemove() { - for (TimelineObjectHolder holder : timeline.findOvershadowed()) { - for (PartitionChunk chunk : holder.getObject()) { + for (TimelineObjectHolder holder : timeline.findFullyOvershadowed()) { + for (PartitionChunk chunk : holder.getObject()) { timeline.remove(holder.getInterval(), holder.getVersion(), chunk); } } - Assert.assertTrue(timeline.findOvershadowed().isEmpty()); + Assert.assertTrue(timeline.findFullyOvershadowed().isEmpty()); } @Test public void testFindEntry() { Assert.assertEquals( - new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle(1))), + new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle("1", 1))), timeline.findEntry(Intervals.of("2011-10-01/2011-10-02"), "1") ); Assert.assertEquals( - new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle(1))), + new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle("1", 1))), timeline.findEntry(Intervals.of("2011-10-01/2011-10-01T10"), "1") ); Assert.assertEquals( - new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle(1))), + new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle("1", 1))), timeline.findEntry(Intervals.of("2011-10-01T02/2011-10-02"), "1") ); Assert.assertEquals( - new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle(1))), + new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle("1", 1))), timeline.findEntry(Intervals.of("2011-10-01T04/2011-10-01T17"), "1") ); @@ -279,7 +283,7 @@ public void testFindEntryWithOverlap() add("2011-01-02/2011-01-05", "2", 1); Assert.assertEquals( - new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle(1))), + new ImmutablePartitionHolder<>(new PartitionHolder<>(makeSingle("1", 1))), timeline.findEntry(Intervals.of("2011-01-02T02/2011-01-04"), "1") ); } @@ -293,8 +297,8 @@ public void testPartitioning() createExpected( "2011-10-02/2011-10-03", "3", Arrays.asList( - IntegerPartitionChunk.make(null, 10, 0, 20), - IntegerPartitionChunk.make(10, null, 1, 21) + IntegerPartitionChunk.make(null, 10, 0, new OvershadowableInteger("3", 0, 20)), + IntegerPartitionChunk.make(10, null, 1, new OvershadowableInteger("3", 1, 21)) ) ), createExpected("2011-10-03/2011-10-04", "3", 3), @@ -310,36 +314,36 @@ public void testPartialPartitionNotReturned() { testRemove(); - add("2011-10-06/2011-10-07", "6", IntegerPartitionChunk.make(null, 10, 0, 60)); + add("2011-10-06/2011-10-07", "6", IntegerPartitionChunk.make(null, 10, 0, new OvershadowableInteger("6", 0, 60))); assertValues( ImmutableList.of(createExpected("2011-10-05/2011-10-06", "5", 5)), timeline.lookup(Intervals.of("2011-10-05/2011-10-07")) ); - Assert.assertTrue("Expected no overshadowed entries", timeline.findOvershadowed().isEmpty()); + Assert.assertTrue("Expected no overshadowed entries", timeline.findFullyOvershadowed().isEmpty()); - add("2011-10-06/2011-10-07", "6", IntegerPartitionChunk.make(10, 20, 1, 61)); + add("2011-10-06/2011-10-07", "6", IntegerPartitionChunk.make(10, 20, 1, new OvershadowableInteger("6", 1, 61))); assertValues( ImmutableList.of(createExpected("2011-10-05/2011-10-06", "5", 5)), timeline.lookup(Intervals.of("2011-10-05/2011-10-07")) ); - Assert.assertTrue("Expected no overshadowed entries", timeline.findOvershadowed().isEmpty()); + Assert.assertTrue("Expected no overshadowed entries", timeline.findFullyOvershadowed().isEmpty()); - add("2011-10-06/2011-10-07", "6", IntegerPartitionChunk.make(20, null, 2, 62)); + add("2011-10-06/2011-10-07", "6", IntegerPartitionChunk.make(20, null, 2, new OvershadowableInteger("6", 2, 62))); assertValues( ImmutableList.of( createExpected("2011-10-05/2011-10-06", "5", 5), createExpected( "2011-10-06/2011-10-07", "6", Arrays.asList( - IntegerPartitionChunk.make(null, 10, 0, 60), - IntegerPartitionChunk.make(10, 20, 1, 61), - IntegerPartitionChunk.make(20, null, 2, 62) + IntegerPartitionChunk.make(null, 10, 0, new OvershadowableInteger("6", 0, 60)), + IntegerPartitionChunk.make(10, 20, 1, new OvershadowableInteger("6", 1, 61)), + IntegerPartitionChunk.make(20, null, 2, new OvershadowableInteger("6", 2, 62)) ) ) ), timeline.lookup(Intervals.of("2011-10-05/2011-10-07")) ); - Assert.assertTrue("Expected no overshadowed entries", timeline.findOvershadowed().isEmpty()); + Assert.assertTrue("Expected no overshadowed entries", timeline.findFullyOvershadowed().isEmpty()); } @Test @@ -347,18 +351,18 @@ public void testIncompletePartitionDoesNotOvershadow() { testRemove(); - add("2011-10-05/2011-10-07", "6", IntegerPartitionChunk.make(null, 10, 0, 60)); - Assert.assertTrue("Expected no overshadowed entries", timeline.findOvershadowed().isEmpty()); + add("2011-10-05/2011-10-07", "6", IntegerPartitionChunk.make(null, 10, 0, new OvershadowableInteger("6", 0, 60))); + Assert.assertTrue("Expected no overshadowed entries", timeline.findFullyOvershadowed().isEmpty()); - add("2011-10-05/2011-10-07", "6", IntegerPartitionChunk.make(10, 20, 1, 61)); - Assert.assertTrue("Expected no overshadowed entries", timeline.findOvershadowed().isEmpty()); + add("2011-10-05/2011-10-07", "6", IntegerPartitionChunk.make(10, 20, 1, new OvershadowableInteger("6", 1, 61))); + Assert.assertTrue("Expected no overshadowed entries", timeline.findFullyOvershadowed().isEmpty()); - add("2011-10-05/2011-10-07", "6", IntegerPartitionChunk.make(20, null, 2, 62)); + add("2011-10-05/2011-10-07", "6", IntegerPartitionChunk.make(20, null, 2, new OvershadowableInteger("6", 2, 62))); assertValues( ImmutableSet.of( createExpected("2011-10-05/2011-10-06", "5", 5) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -367,13 +371,18 @@ public void testRemovePartitionMakesIncomplete() { testIncompletePartitionDoesNotOvershadow(); - final IntegerPartitionChunk chunk = IntegerPartitionChunk.make(null, 10, 0, 60); + final IntegerPartitionChunk chunk = IntegerPartitionChunk.make( + null, + 10, + 0, + new OvershadowableInteger("6", 0, 60) + ); Assert.assertEquals(chunk, timeline.remove(Intervals.of("2011-10-05/2011-10-07"), "6", chunk)); assertValues( ImmutableList.of(createExpected("2011-10-05/2011-10-06", "5", 5)), timeline.lookup(Intervals.of("2011-10-05/2011-10-07")) ); - Assert.assertTrue("Expected no overshadowed entries", timeline.findOvershadowed().isEmpty()); + Assert.assertTrue("Expected no overshadowed entries", timeline.findFullyOvershadowed().isEmpty()); } @Test @@ -388,8 +397,8 @@ public void testInsertAndRemoveSameThingsion() ); Assert.assertEquals( - makeSingle(10), - timeline.remove(Intervals.of("2011-05-01/2011-05-10"), "5", makeSingle(10)) + makeSingle("5", 10), + timeline.remove(Intervals.of("2011-05-01/2011-05-10"), "5", makeSingle("5", 10)) ); assertValues( Collections.singletonList( @@ -407,8 +416,8 @@ public void testInsertAndRemoveSameThingsion() ); Assert.assertEquals( - makeSingle(9), - timeline.remove(Intervals.of("2011-05-01/2011-05-10"), "4", makeSingle(9)) + makeSingle("4", 9), + timeline.remove(Intervals.of("2011-05-01/2011-05-10"), "4", makeSingle("4", 9)) ); assertValues( Collections.singletonList( @@ -429,6 +438,8 @@ public void testOverlapSameVersionThrowException() add("2011-01-05/2011-01-15", "1", 3); } + // 2|----| + // 2|----| // 1|----| // 1|----| @Test @@ -560,8 +571,8 @@ public void testOverlapSecondAfter() assertValues( Arrays.asList( - createExpected("2011-01-01/2011-01-10", "2", 1), - createExpected("2011-01-10/2011-01-15", "1", 3) + createExpected("2011-01-01/2011-01-10", "2", 3), + createExpected("2011-01-10/2011-01-15", "1", 1) ), timeline.lookup(Intervals.of("2011-01-01/2011-01-15")) ); @@ -930,7 +941,7 @@ public void testOverlapV1Large() createExpected("2011-01-01/2011-01-03", "1", 1), createExpected("2011-01-03/2011-01-05", "2", 2), createExpected("2011-01-05/2011-01-13", "1", 1), - createExpected("2011-01-13/2011-01-20", "2", 2) + createExpected("2011-01-13/2011-01-20", "2", 3) ), timeline.lookup(Intervals.of("2011-01-01/2011-01-20")) ); @@ -949,8 +960,8 @@ public void testOverlapV2Large() assertValues( Arrays.asList( - createExpected("2011-01-01/2011-01-15", "2", 2), - createExpected("2011-01-15/2011-01-20", "1", 1) + createExpected("2011-01-01/2011-01-15", "2", 1), + createExpected("2011-01-15/2011-01-20", "1", 3) ), timeline.lookup(Intervals.of("2011-01-01/2011-01-20")) ); @@ -1109,7 +1120,7 @@ public void testOverlapOvershadowedThirdContains() createExpected("2011-01-03/2011-01-06", "1", 1), createExpected("2011-01-09/2011-01-12", "1", 2) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1132,7 +1143,7 @@ public void testOverlapOvershadowedAligned() createExpected("2011-01-05/2011-01-10", "2", 2), createExpected("2011-01-01/2011-01-10", "1", 3) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1154,7 +1165,7 @@ public void testOverlapOvershadowedSomeComplexOverlapsCantThinkOfBetterName() createExpected("2011-01-03/2011-01-12", "1", 3), createExpected("2011-01-01/2011-01-05", "2", 1) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1166,7 +1177,7 @@ public void testOverlapAndRemove() add("2011-01-01/2011-01-20", "1", 1); add("2011-01-10/2011-01-15", "2", 2); - timeline.remove(Intervals.of("2011-01-10/2011-01-15"), "2", makeSingle(2)); + timeline.remove(Intervals.of("2011-01-10/2011-01-15"), "2", makeSingle("2", 2)); assertValues( Collections.singletonList( @@ -1185,7 +1196,7 @@ public void testOverlapAndRemove2() add("2011-01-10/2011-01-20", "2", 2); add("2011-01-20/2011-01-30", "3", 4); - timeline.remove(Intervals.of("2011-01-10/2011-01-20"), "2", makeSingle(2)); + timeline.remove(Intervals.of("2011-01-10/2011-01-20"), "2", makeSingle("2", 2)); assertValues( Arrays.asList( @@ -1206,8 +1217,8 @@ public void testOverlapAndRemove3() add("2011-01-02/2011-01-03", "2", 2); add("2011-01-10/2011-01-14", "2", 3); - timeline.remove(Intervals.of("2011-01-02/2011-01-03"), "2", makeSingle(2)); - timeline.remove(Intervals.of("2011-01-10/2011-01-14"), "2", makeSingle(3)); + timeline.remove(Intervals.of("2011-01-02/2011-01-03"), "2", makeSingle("2", 2)); + timeline.remove(Intervals.of("2011-01-10/2011-01-14"), "2", makeSingle("2", 3)); assertValues( Collections.singletonList( @@ -1227,7 +1238,7 @@ public void testOverlapAndRemove4() add("2011-01-10/2011-01-15", "2", 2); add("2011-01-15/2011-01-20", "2", 3); - timeline.remove(Intervals.of("2011-01-15/2011-01-20"), "2", makeSingle(3)); + timeline.remove(Intervals.of("2011-01-15/2011-01-20"), "2", makeSingle("2", 3)); assertValues( Arrays.asList( @@ -1246,7 +1257,7 @@ public void testOverlapAndRemove5() add("2011-01-01/2011-01-20", "1", 1); add("2011-01-10/2011-01-15", "2", 2); - timeline.remove(Intervals.of("2011-01-10/2011-01-15"), "2", makeSingle(2)); + timeline.remove(Intervals.of("2011-01-10/2011-01-15"), "2", makeSingle("2", 2)); add("2011-01-01/2011-01-20", "1", 1); assertValues( @@ -1262,11 +1273,11 @@ public void testRemoveSomethingDontHave() { Assert.assertNull( "Don't have it, should be null", - timeline.remove(Intervals.of("1970-01-01/2025-04-20"), "1", makeSingle(1)) + timeline.remove(Intervals.of("1970-01-01/2025-04-20"), "1", makeSingle("1", 1)) ); Assert.assertNull( "Don't have it, should be null", - timeline.remove(Intervals.of("2011-04-01/2011-04-09"), "version does not exist", makeSingle(1)) + timeline.remove(Intervals.of("2011-04-01/2011-04-09"), "version does not exist", makeSingle("version does not exist", 1)) ); } @@ -1280,7 +1291,7 @@ public void testRemoveNothingBacking() add("2011-01-10/2011-01-15", "3", 3); add("2011-01-15/2011-01-20", "4", 4); - timeline.remove(Intervals.of("2011-01-15/2011-01-20"), "4", makeSingle(4)); + timeline.remove(Intervals.of("2011-01-15/2011-01-20"), "4", makeSingle("4", 4)); assertValues( Arrays.asList( @@ -1308,7 +1319,7 @@ public void testOvershadowingHigherVersionWins1() createExpected("2011-04-03/2011-04-06", "1", 3), createExpected("2011-04-06/2011-04-09", "1", 4) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1326,7 +1337,7 @@ public void testOvershadowingHigherVersionWins2() ImmutableSet.of( createExpected("2011-04-01/2011-04-09", "1", 1) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1346,7 +1357,7 @@ public void testOvershadowingHigherVersionWins3() createExpected("2011-04-03/2011-04-06", "1", 3), createExpected("2011-04-09/2011-04-12", "1", 4) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1364,7 +1375,7 @@ public void testOvershadowingHigherVersionWins4() createExpected("2011-04-03/2011-04-06", "1", 3), createExpected("2011-04-06/2011-04-09", "1", 4) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1379,7 +1390,7 @@ public void testOvershadowingHigherVersionNeverOvershadowedByLower1() assertValues( ImmutableSet.of(), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1394,7 +1405,7 @@ public void testOvershadowingHigherVersionNeverOvershadowedByLower2() assertValues( ImmutableSet.of(), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1409,7 +1420,7 @@ public void testOvershadowingHigherVersionNeverOvershadowedByLower3() assertValues( ImmutableSet.of(), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1427,7 +1438,7 @@ public void testOvershadowingHigherVersionNeverOvershadowedByLower4() ImmutableSet.of( createExpected("2011-04-03/2011-04-06", "1", 3) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1447,7 +1458,7 @@ public void testOvershadowingHigherVersionNeverOvershadowedByLower5() createExpected("2011-04-03/2011-04-06", "1", 3), createExpected("2011-04-09/2011-04-12", "1", 3) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1465,7 +1476,7 @@ public void testOvershadowingSameIntervalHighVersionWins() createExpected("2011-04-01/2011-04-09", "2", 3), createExpected("2011-04-01/2011-04-09", "1", 1) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1484,7 +1495,7 @@ public void testOvershadowingSameIntervalSameVersionAllKept() createExpected("2011-04-01/2011-04-09", "2", 3), createExpected("2011-04-01/2011-04-09", "1", 1) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); } @@ -1504,30 +1515,30 @@ public void testRemoveIncompleteKeepsComplete() { timeline = makeStringIntegerTimeline(); - add("2011-04-01/2011-04-02", "1", IntegerPartitionChunk.make(null, 1, 0, 77)); - add("2011-04-01/2011-04-02", "1", IntegerPartitionChunk.make(1, null, 1, 88)); - add("2011-04-01/2011-04-02", "2", IntegerPartitionChunk.make(null, 1, 0, 99)); + add("2011-04-01/2011-04-02", "1", IntegerPartitionChunk.make(null, 1, 0, new OvershadowableInteger("1", 0, 77))); + add("2011-04-01/2011-04-02", "1", IntegerPartitionChunk.make(1, null, 1, new OvershadowableInteger("1", 1, 88))); + add("2011-04-01/2011-04-02", "2", IntegerPartitionChunk.make(null, 1, 0, new OvershadowableInteger("2", 0, 99))); assertValues( ImmutableList.of( createExpected("2011-04-01/2011-04-02", "1", Arrays.asList( - IntegerPartitionChunk.make(null, 1, 0, 77), - IntegerPartitionChunk.make(1, null, 1, 88) + IntegerPartitionChunk.make(null, 1, 0, new OvershadowableInteger("1", 0, 77)), + IntegerPartitionChunk.make(1, null, 1, new OvershadowableInteger("1", 1, 88)) ) ) ), timeline.lookup(Intervals.of("2011-04-01/2011-04-02")) ); - add("2011-04-01/2011-04-02", "3", IntegerPartitionChunk.make(null, 1, 0, 110)); + add("2011-04-01/2011-04-02", "3", IntegerPartitionChunk.make(null, 1, 0, new OvershadowableInteger("3", 0, 110))); assertValues( ImmutableList.of( createExpected("2011-04-01/2011-04-02", "1", Arrays.asList( - IntegerPartitionChunk.make(null, 1, 0, 77), - IntegerPartitionChunk.make(1, null, 1, 88) + IntegerPartitionChunk.make(null, 1, 0, new OvershadowableInteger("1", 0, 77)), + IntegerPartitionChunk.make(1, null, 1, new OvershadowableInteger("1", 1, 88)) ) ) ), @@ -1537,11 +1548,11 @@ public void testRemoveIncompleteKeepsComplete() Sets.newHashSet( createExpected("2011-04-01/2011-04-02", "2", Collections.singletonList( - IntegerPartitionChunk.make(null, 1, 0, 99) + IntegerPartitionChunk.make(null, 1, 0, new OvershadowableInteger("2", 0, 99)) ) ) ), - timeline.findOvershadowed() + timeline.findFullyOvershadowed() ); testRemove(); @@ -1550,8 +1561,8 @@ public void testRemoveIncompleteKeepsComplete() ImmutableList.of( createExpected("2011-04-01/2011-04-02", "1", Arrays.asList( - IntegerPartitionChunk.make(null, 1, 0, 77), - IntegerPartitionChunk.make(1, null, 1, 88) + IntegerPartitionChunk.make(null, 1, 0, new OvershadowableInteger("1", 0, 77)), + IntegerPartitionChunk.make(1, null, 1, new OvershadowableInteger("1", 1, 88)) ) ) ), @@ -1564,64 +1575,64 @@ public void testIsOvershadowedWithNonOverlappingSegmentsInTimeline() { timeline = makeStringIntegerTimeline(); - add("2011-04-05/2011-04-07", "1", new SingleElementPartitionChunk<>(1)); - add("2011-04-07/2011-04-09", "1", new SingleElementPartitionChunk<>(1)); - - add("2011-04-15/2011-04-17", "1", new SingleElementPartitionChunk<>(1)); - add("2011-04-17/2011-04-19", "1", new SingleElementPartitionChunk<>(1)); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-03"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-05"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-06"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-07"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-08"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-09"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-10"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-30"), "0")); - - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "1")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "1")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "1")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "1")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "2")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "2")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "2")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "2")); - - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-07"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-08"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-09"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-10"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-30"), "0")); - - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-07/2011-04-08"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-07/2011-04-09"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-07/2011-04-10"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-07/2011-04-30"), "0")); - - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-08/2011-04-09"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-08/2011-04-10"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-08/2011-04-30"), "0")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-10"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-15"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-17"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-19"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-30"), "0")); - - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-16"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-17"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-18"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-19"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-20"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-30"), "0")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-19/2011-04-20"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-21/2011-04-22"), "0")); + add("2011-04-05/2011-04-07", "1", makeSingle("1", 1)); + add("2011-04-07/2011-04-09", "1", makeSingle("1", 1)); + + add("2011-04-15/2011-04-17", "1", makeSingle("1", 1)); + add("2011-04-17/2011-04-19", "1", makeSingle("1", 1)); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-03"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-05"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-06"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-07"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-08"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-09"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-10"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-30"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "1", new OvershadowableInteger("1", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "1", new OvershadowableInteger("1", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "1", new OvershadowableInteger("1", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "1", new OvershadowableInteger("1", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "2", new OvershadowableInteger("2", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "2", new OvershadowableInteger("2", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "2", new OvershadowableInteger("2", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "2", new OvershadowableInteger("2", 0, 1))); + + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-07"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-08"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-09"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-10"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-30"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-07/2011-04-08"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-07/2011-04-09"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-07/2011-04-10"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-07/2011-04-30"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-08/2011-04-09"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-08/2011-04-10"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-08/2011-04-30"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-10"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-15"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-17"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-19"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-09/2011-04-30"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-16"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-17"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-18"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-19"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-20"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-30"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-19/2011-04-20"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-21/2011-04-22"), "0", new OvershadowableInteger("0", 0, 1))); } @Test @@ -1629,87 +1640,370 @@ public void testIsOvershadowedWithOverlappingSegmentsInTimeline() { timeline = makeStringIntegerTimeline(); - add("2011-04-05/2011-04-09", "11", new SingleElementPartitionChunk<>(1)); - add("2011-04-07/2011-04-11", "12", new SingleElementPartitionChunk<>(1)); - - add("2011-04-15/2011-04-19", "12", new SingleElementPartitionChunk<>(1)); - add("2011-04-17/2011-04-21", "11", new SingleElementPartitionChunk<>(1)); - - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-03"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-05"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-06"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-07"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-08"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-09"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-10"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-11"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-30"), "0")); - - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-10"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-11"), "0")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "12")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "12")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "12")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "12")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-10"), "12")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-11"), "12")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "13")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "13")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "13")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "13")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-10"), "13")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-11"), "13")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-12"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-15"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-16"), "0")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-17"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-18"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-19"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-20"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-21"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-22"), "0")); - - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-07"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-08"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-09"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-10"), "0")); - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-11"), "0")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-12"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-15"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-16"), "0")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-17"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-18"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-19"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-20"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-21"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-22"), "0")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-15"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-16"), "0")); - - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-17"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-18"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-19"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-20"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-21"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-22"), "0")); - - Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-21"), "0")); - Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-21/2011-04-22"), "0")); - } - - private Pair>> createExpected( + add("2011-04-05/2011-04-09", "11", makeSingle("11", 1)); + add("2011-04-07/2011-04-11", "12", makeSingle("12", 1)); + + add("2011-04-15/2011-04-19", "12", makeSingle("12", 1)); + add("2011-04-17/2011-04-21", "11", makeSingle("11", 1)); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-03"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-05"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-06"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-07"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-08"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-09"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-10"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-11"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-01/2011-04-30"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-10"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-11"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "12", new OvershadowableInteger("12", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "12", new OvershadowableInteger("12", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "12", new OvershadowableInteger("12", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "12", new OvershadowableInteger("12", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-10"), "12", new OvershadowableInteger("12", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-11"), "12", new OvershadowableInteger("12", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-06"), "13", new OvershadowableInteger("13", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-07"), "13", new OvershadowableInteger("13", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-08"), "13", new OvershadowableInteger("13", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-09"), "13", new OvershadowableInteger("13", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-10"), "13", new OvershadowableInteger("13", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-11"), "13", new OvershadowableInteger("13", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-12"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-15"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-16"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-17"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-18"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-19"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-20"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-21"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-05/2011-04-22"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-07"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-08"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-09"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-10"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-11"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-12"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-15"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-16"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-17"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-18"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-19"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-20"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-21"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-06/2011-04-22"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-15"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-16"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-17"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-18"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-19"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-20"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-21"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-12/2011-04-22"), "0", new OvershadowableInteger("0", 0, 1))); + + Assert.assertTrue(timeline.isOvershadowed(Intervals.of("2011-04-15/2011-04-21"), "0", new OvershadowableInteger("0", 0, 1))); + Assert.assertFalse(timeline.isOvershadowed(Intervals.of("2011-04-21/2011-04-22"), "0", new OvershadowableInteger("0", 0, 1))); + } + + @Test + public void testOvershadowedByReference() + { + timeline = makeStringIntegerTimeline(); + + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 0, 0)); + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 1, 0)); + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 2, 0)); + + add("2019-01-01/2019-01-02", "0", makeNumberedOverwriting("0", 0, 1, 0, 3, 1, 2)); + add("2019-01-01/2019-01-02", "0", makeNumberedOverwriting("0", 1, 1, 0, 3, 1, 2)); + + Assert.assertEquals( + ImmutableSet.of( + makeTimelineObjectHolder( + "2019-01-01/2019-01-02", + "0", + ImmutableList.of(makeNumbered("0", 0, 0), makeNumbered("0", 1, 0), makeNumbered("0", 2, 0)) + ) + ), + timeline.findFullyOvershadowed() + ); + } + + @Test + public void testOvershadowedByReferenceChain() + { + timeline = makeStringIntegerTimeline(); + + // 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 0, 0)); + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 1, 0)); + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 2, 0)); + + // 2019-01-02/2019-01-03 + add("2019-01-02/2019-01-03", "0", makeNumbered("0", 0, 0)); + add("2019-01-02/2019-01-03", "0", makeNumbered("0", 1, 0)); + + // Overwrite 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "0", makeNumberedOverwriting("0", 0, 1, 0, 3, 1, 2)); + add("2019-01-01/2019-01-02", "0", makeNumberedOverwriting("0", 1, 1, 0, 3, 1, 2)); + + // Overwrite 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "0", makeNumberedOverwriting("0", 2, 2, 0, 3, 2, 2)); + add("2019-01-01/2019-01-02", "0", makeNumberedOverwriting("0", 3, 2, 0, 3, 2, 2)); + + Assert.assertEquals( + ImmutableSet.of( + makeTimelineObjectHolder( + "2019-01-01/2019-01-02", + "0", + ImmutableList.of( + makeNumbered("0", 0, 0), + makeNumbered("0", 1, 0), + makeNumbered("0", 2, 0), + makeNumberedOverwriting("0", 0, 1, 0, 3, 1, 2), + makeNumberedOverwriting("0", 1, 1, 0, 3, 1, 2) + ) + ) + ), + timeline.findFullyOvershadowed() + ); + } + + @Test + public void testOvershadowedByReferenceAndThenVersion() + { + timeline = makeStringIntegerTimeline(); + + // 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 0, 0)); + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 1, 0)); + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 2, 0)); + + // 2019-01-02/2019-01-03 + add("2019-01-02/2019-01-03", "0", makeNumbered("0", 0, 0)); + add("2019-01-02/2019-01-03", "0", makeNumbered("0", 1, 0)); + + // Overwrite 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "0", makeNumberedOverwriting("0", 0, 1, 0, 3, 1, 2)); + add("2019-01-01/2019-01-02", "0", makeNumberedOverwriting("0", 1, 1, 0, 3, 1, 2)); + + // Overwrite 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "1", makeNumbered("1", 0, 0)); + add("2019-01-01/2019-01-02", "1", makeNumbered("1", 1, 0)); + + Assert.assertEquals( + ImmutableSet.of( + makeTimelineObjectHolder( + "2019-01-01/2019-01-02", + "0", + ImmutableList.of( + makeNumbered("0", 0, 0), + makeNumbered("0", 1, 0), + makeNumbered("0", 2, 0), + makeNumberedOverwriting("0", 0, 1, 0, 3, 1, 2), + makeNumberedOverwriting("0", 1, 1, 0, 3, 1, 2) + ) + ) + ), + timeline.findFullyOvershadowed() + ); + } + + @Test + public void testOvershadowedByVersionAndThenReference() + { + timeline = makeStringIntegerTimeline(); + + // 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 0, 0)); + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 1, 0)); + add("2019-01-01/2019-01-02", "0", makeNumbered("0", 2, 0)); + + // 2019-01-02/2019-01-03 + add("2019-01-02/2019-01-03", "0", makeNumbered("0", 0, 0)); + add("2019-01-02/2019-01-03", "0", makeNumbered("0", 1, 0)); + + // Overwrite 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "1", makeNumbered("1", 0, 0)); + add("2019-01-01/2019-01-02", "1", makeNumbered("1", 1, 0)); + + // Overwrite 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "1", makeNumberedOverwriting("1", 0, 1, 0, 2, 1, 3)); + add("2019-01-01/2019-01-02", "1", makeNumberedOverwriting("1", 1, 1, 0, 2, 1, 3)); + add("2019-01-01/2019-01-02", "1", makeNumberedOverwriting("1", 2, 1, 0, 2, 1, 3)); + + Assert.assertEquals( + ImmutableSet.of( + makeTimelineObjectHolder( + "2019-01-01/2019-01-02", + "0", + ImmutableList.of( + makeNumbered("0", 0, 0), + makeNumbered("0", 1, 0), + makeNumbered("0", 2, 0) + ) + ), + makeTimelineObjectHolder( + "2019-01-01/2019-01-02", + "1", + ImmutableList.of( + makeNumbered("1", 0, 0), + makeNumbered("1", 1, 0) + ) + ) + ), + timeline.findFullyOvershadowed() + ); + } + + @Test + public void testFallbackOnMissingSegment() + { + timeline = makeStringIntegerTimeline(); + + final Interval interval = Intervals.of("2019-01-01/2019-01-02"); + + add(interval, "0", makeNumbered("0", 0, 0)); + add(interval, "0", makeNumbered("0", 1, 0)); + add(interval, "0", makeNumbered("0", 2, 0)); + + // Overwrite 2019-01-01/2019-01-02 + add(interval, "1", makeNumbered("1", 0, 0)); + add(interval, "1", makeNumbered("1", 1, 0)); + + // Overwrite 2019-01-01/2019-01-02 + add("2019-01-01/2019-01-02", "1", makeNumberedOverwriting("1", 0, 1, 0, 2, 1, 3)); + add("2019-01-01/2019-01-02", "1", makeNumberedOverwriting("1", 1, 1, 0, 2, 1, 3)); + add("2019-01-01/2019-01-02", "1", makeNumberedOverwriting("1", 2, 1, 0, 2, 1, 3)); + + timeline.remove( + interval, + "1", + makeNumberedOverwriting("1", 2, 1, 0, 2, 1, 3) + ); + + final List> holders = timeline.lookup(interval); + + Assert.assertEquals( + ImmutableList.of( + new TimelineObjectHolder<>( + interval, + "1", + new PartitionHolder<>( + ImmutableList.of( + makeNumbered("1", 0, 0), + makeNumbered("1", 1, 0), + makeNumberedOverwriting("1", 0, 1, 0, 2, 1, 3), + makeNumberedOverwriting("1", 1, 1, 0, 2, 1, 3) + ) + ) + ) + ), + holders + ); + } + + @Test + public void testAddSameChunkToFullAtomicUpdateGroup() + { + timeline = makeStringIntegerTimeline(); + final Interval interval = Intervals.of("2019-01-01/2019-01-02"); + add(interval, "0", makeNumbered("0", 0, 0)); + add(interval, "0", makeNumberedOverwriting("0", 0, 0, 0, 1, 1, 1)); + add(interval, "0", makeNumbered("0", 0, 1)); + + final Set> overshadowed = timeline.findFullyOvershadowed(); + Assert.assertEquals( + ImmutableSet.of( + new TimelineObjectHolder<>( + interval, + "0", + new PartitionHolder<>(ImmutableList.of(makeNumbered("0", 0, 1))) + ) + ), + overshadowed + ); + } + + @Test + public void testOvershadowMultipleStandbyAtomicUpdateGroup() + { + timeline = makeStringIntegerTimeline(); + final Interval interval = Intervals.of("2019-01-01/2019-01-02"); + add(interval, "0", makeNumberedOverwriting("0", 0, 0, 0, 1, 1, 2)); + add(interval, "0", makeNumberedOverwriting("0", 1, 0, 0, 1, 2, 2)); + add(interval, "0", makeNumberedOverwriting("0", 2, 0, 0, 1, 3, 2)); // <-- full atomicUpdateGroup + add(interval, "0", makeNumberedOverwriting("0", 3, 1, 0, 1, 3, 2)); // <-- full atomicUpdateGroup + + final Set> overshadowed = timeline.findFullyOvershadowed(); + Assert.assertEquals( + ImmutableSet.of( + new TimelineObjectHolder<>( + interval, + "0", + new PartitionHolder<>( + ImmutableList.of( + makeNumberedOverwriting("0", 0, 0, 0, 1, 1, 2), + makeNumberedOverwriting("0", 1, 0, 0, 1, 2, 2) + ) + ) + ) + ), + overshadowed + ); + } + + @Test + public void testIsOvershadowedForOverwritingSegments() + { + timeline = makeStringIntegerTimeline(); + final Interval interval = Intervals.of("2019-01-01/2019-01-02"); + add(interval, "0", makeNumberedOverwriting("0", 0, 0, 5, 10, 10, 1)); + + for (int i = 0; i < 5; i++) { + Assert.assertTrue(timeline.isOvershadowed(interval, "0", makeNumbered("0", i + 5, 0).getObject())); + } + + Assert.assertFalse(timeline.isOvershadowed(interval, "0", makeNumbered("0", 4, 0).getObject())); + Assert.assertFalse(timeline.isOvershadowed(interval, "0", makeNumbered("0", 11, 0).getObject())); + + Assert.assertTrue(timeline.isOvershadowed(interval, "0", makeNumberedOverwriting("0", 1, 0, 5, 6, 5, 2).getObject())); + Assert.assertTrue(timeline.isOvershadowed(interval, "0", makeNumberedOverwriting("0", 1, 0, 7, 8, 5, 2).getObject())); + Assert.assertTrue(timeline.isOvershadowed(interval, "0", makeNumberedOverwriting("0", 1, 0, 8, 10, 5, 2).getObject())); + + Assert.assertFalse(timeline.isOvershadowed(interval, "0", makeNumberedOverwriting("0", 1, 0, 5, 10, 12, 2).getObject())); + Assert.assertFalse(timeline.isOvershadowed(interval, "0", makeNumberedOverwriting("0", 1, 0, 4, 15, 12, 2).getObject())); + } + + private TimelineObjectHolder makeTimelineObjectHolder( + String interval, + String version, + List> chunks + ) + { + return new TimelineObjectHolder<>( + Intervals.of(interval), + Intervals.of(interval), + version, + new PartitionHolder<>(chunks) + ); + } + + private Pair>> createExpected( String intervalString, String version, Integer value @@ -1718,14 +2012,14 @@ private Pair>> createExpected( return createExpected( intervalString, version, - Collections.singletonList(makeSingle(value)) + Collections.singletonList(makeSingle(version, value)) ); } - private Pair>> createExpected( + private Pair>> createExpected( String intervalString, String version, - List> values + List> values ) { return Pair.of( @@ -1734,9 +2028,48 @@ private Pair>> createExpected( ); } - private SingleElementPartitionChunk makeSingle(Integer value) + private PartitionChunk makeSingle(String majorVersion, int value) + { + return makeSingle(majorVersion, 0, value); + } + + private PartitionChunk makeSingle(String majorVersion, int partitionNum, int val) { - return new SingleElementPartitionChunk<>(value); + return new SingleElementPartitionChunk<>(new OvershadowableInteger(majorVersion, partitionNum, val)); + } + + private PartitionChunk makeNumbered(String majorVersion, int partitionNum, int val) + { + return new NumberedPartitionChunk<>( + partitionNum, + 0, + new OvershadowableInteger(majorVersion, partitionNum, val) + ); + } + + private PartitionChunk makeNumberedOverwriting( + String majorVersion, + int partitionNumOrdinal, + int val, + int startRootPartitionId, + int endRootPartitionId, + int minorVersion, + int atomicUpdateGroupSize + ) + { + final int partitionNum = PartitionIds.NON_ROOT_GEN_START_PARTITION_ID + partitionNumOrdinal; + return new NumberedOverwritingPartitionChunk<>( + partitionNum, + new OvershadowableInteger( + majorVersion, + partitionNum, + val, + startRootPartitionId, + endRootPartitionId, + minorVersion, + atomicUpdateGroupSize + ) + ); } private void add(String interval, String version, Integer value) @@ -1746,69 +2079,182 @@ private void add(String interval, String version, Integer value) private void add(Interval interval, String version, Integer value) { - add(interval, version, makeSingle(value)); + add(interval, version, makeSingle(version, value)); } - private void add(String interval, String version, PartitionChunk value) + private void add(String interval, String version, PartitionChunk value) { add(Intervals.of(interval), version, value); } - private void add(Interval interval, String version, PartitionChunk value) + private void add(Interval interval, String version, PartitionChunk value) { timeline.add(interval, version, value); } private void assertValues( - List>>> expected, - List> actual + List>>> expected, + List> actual ) { Assert.assertEquals("Sizes did not match.", expected.size(), actual.size()); - Iterator>>> expectedIter = expected.iterator(); - Iterator> actualIter = actual.iterator(); + Iterator>>> expectedIter = expected.iterator(); + Iterator> actualIter = actual.iterator(); while (expectedIter.hasNext()) { - Pair>> pair = expectedIter.next(); - TimelineObjectHolder holder = actualIter.next(); + Pair>> pair = expectedIter.next(); + TimelineObjectHolder holder = actualIter.next(); Assert.assertEquals(pair.lhs, holder.getInterval()); Assert.assertEquals(pair.rhs.lhs, holder.getVersion()); - Assert.assertEquals(pair.rhs.rhs, holder.getObject()); + + final List> expectedChunks = Lists.newArrayList(pair.rhs.rhs); + final List> actualChunks = Lists.newArrayList(holder.getObject()); + + Assert.assertEquals(expectedChunks.size(), actualChunks.size()); + for (int i = 0; i < expectedChunks.size(); i++) { + // Check partitionNumber first + Assert.assertEquals(expectedChunks.get(i), actualChunks.get(i)); + final OvershadowableInteger expectedInteger = expectedChunks.get(i).getObject(); + final OvershadowableInteger actualInteger = actualChunks.get(i).getObject(); + Assert.assertEquals(expectedInteger, actualInteger); + } } } private void assertValues( - Set>>> expected, - Set> actual + Set>>> expected, + Set> actual ) { Assert.assertEquals("Sizes did not match.", expected.size(), actual.size()); - Set>>> actualSet = + Set>>> actualSet = Sets.newHashSet( Iterables.transform( actual, - new Function, Pair>>>() - { - @Override - public Pair>> apply( - TimelineObjectHolder input - ) - { - return new Pair<>(input.getInterval(), new Pair<>(input.getVersion(), input.getObject())); - } - } + input -> new Pair<>(input.getInterval(), new Pair<>(input.getVersion(), input.getObject())) ) ); Assert.assertEquals(expected, actualSet); } - private VersionedIntervalTimeline makeStringIntegerTimeline() + private VersionedIntervalTimeline makeStringIntegerTimeline() { return new VersionedIntervalTimeline<>(Ordering.natural()); } + private static class OvershadowableInteger implements Overshadowable + { + private final String majorVersion; + private final int partitionNum; + private final int val; + private final int startRootPartitionId; + private final int endRootPartitionId; + private final short minorVersion; + private final short atomicUpdateGroupSize; + + private OvershadowableInteger(String majorVersion, int partitionNum, int val) + { + this(majorVersion, partitionNum, val, partitionNum, partitionNum + 1, 0, 1); + } + + private OvershadowableInteger( + String majorVersion, + int partitionNum, + int val, + int startRootPartitionId, + int endRootPartitionId, + int minorVersion, + int atomicUpdateGroupSize + ) + { + this.majorVersion = majorVersion; + this.partitionNum = partitionNum; + this.val = val; + this.startRootPartitionId = startRootPartitionId; + this.endRootPartitionId = endRootPartitionId; + this.minorVersion = (short) minorVersion; + this.atomicUpdateGroupSize = (short) atomicUpdateGroupSize; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OvershadowableInteger that = (OvershadowableInteger) o; + return partitionNum == that.partitionNum && + val == that.val && + startRootPartitionId == that.startRootPartitionId && + endRootPartitionId == that.endRootPartitionId && + minorVersion == that.minorVersion && + atomicUpdateGroupSize == that.atomicUpdateGroupSize && + Objects.equals(majorVersion, that.majorVersion); + } + + @Override + public int hashCode() + { + return Objects.hash( + majorVersion, + partitionNum, + val, + startRootPartitionId, + endRootPartitionId, + minorVersion, + atomicUpdateGroupSize + ); + } + + @Override + public String toString() + { + return "OvershadowableInteger{" + + "majorVersion='" + majorVersion + '\'' + + ", partitionNum=" + partitionNum + + ", val=" + val + + ", startRootPartitionId=" + startRootPartitionId + + ", endRootPartitionId=" + endRootPartitionId + + ", minorVersion=" + minorVersion + + ", atomicUpdateGroupSize=" + atomicUpdateGroupSize + + '}'; + } + + @Override + public int getStartRootPartitionId() + { + return startRootPartitionId; + } + + @Override + public int getEndRootPartitionId() + { + return endRootPartitionId; + } + + @Override + public String getVersion() + { + return majorVersion; + } + + @Override + public short getMinorVersion() + { + return minorVersion; + } + + @Override + public short getAtomicUpdateGroupSize() + { + return atomicUpdateGroupSize; + } + } } diff --git a/core/src/test/java/org/apache/druid/timeline/partition/IntegerPartitionChunkTest.java b/core/src/test/java/org/apache/druid/timeline/partition/IntegerPartitionChunkTest.java index 36b3915f9dec..04e4ccf4cae6 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/IntegerPartitionChunkTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/IntegerPartitionChunkTest.java @@ -19,86 +19,135 @@ package org.apache.druid.timeline.partition; +import org.apache.druid.timeline.Overshadowable; import org.junit.Assert; import org.junit.Test; public class IntegerPartitionChunkTest { + private static IntegerPartitionChunk make( + Integer start, + Integer end, + int chunkNumber, + int obj + ) + { + return new IntegerPartitionChunk<>(start, end, chunkNumber, new OvershadowableInteger(obj)); + } + @Test public void testAbuts() { - IntegerPartitionChunk lhs = IntegerPartitionChunk.make(null, 10, 0, 1); + IntegerPartitionChunk lhs = make(null, 10, 0, 1); - Assert.assertTrue(lhs.abuts(IntegerPartitionChunk.make(10, null, 1, 2))); - Assert.assertFalse(lhs.abuts(IntegerPartitionChunk.make(11, null, 2, 3))); - Assert.assertFalse(lhs.abuts(IntegerPartitionChunk.make(null, null, 3, 4))); + Assert.assertTrue(lhs.abuts(make(10, null, 1, 2))); + Assert.assertFalse(lhs.abuts(make(11, null, 2, 3))); + Assert.assertFalse(lhs.abuts(make(null, null, 3, 4))); - Assert.assertFalse(IntegerPartitionChunk.make(null, null, 0, 1) - .abuts(IntegerPartitionChunk.make(null, null, 1, 2))); + Assert.assertFalse(make(null, null, 0, 1).abuts(make(null, null, 1, 2))); } @Test public void testIsStart() { - Assert.assertTrue(IntegerPartitionChunk.make(null, 10, 0, 1).isStart()); - Assert.assertFalse(IntegerPartitionChunk.make(10, null, 0, 1).isStart()); - Assert.assertFalse(IntegerPartitionChunk.make(10, 11, 0, 1).isStart()); - Assert.assertTrue(IntegerPartitionChunk.make(null, null, 0, 1).isStart()); + Assert.assertTrue(make(null, 10, 0, 1).isStart()); + Assert.assertFalse(make(10, null, 0, 1).isStart()); + Assert.assertFalse(make(10, 11, 0, 1).isStart()); + Assert.assertTrue(make(null, null, 0, 1).isStart()); } @Test public void testIsEnd() { - Assert.assertFalse(IntegerPartitionChunk.make(null, 10, 0, 1).isEnd()); - Assert.assertTrue(IntegerPartitionChunk.make(10, null, 0, 1).isEnd()); - Assert.assertFalse(IntegerPartitionChunk.make(10, 11, 0, 1).isEnd()); - Assert.assertTrue(IntegerPartitionChunk.make(null, null, 0, 1).isEnd()); + Assert.assertFalse(make(null, 10, 0, 1).isEnd()); + Assert.assertTrue(make(10, null, 0, 1).isEnd()); + Assert.assertFalse(make(10, 11, 0, 1).isEnd()); + Assert.assertTrue(make(null, null, 0, 1).isEnd()); } @Test public void testCompareTo() { - //noinspection EqualsWithItself (the intention of this first test is specifically to call compareTo with itself) Assert.assertEquals( 0, - IntegerPartitionChunk.make(null, null, 0, 1).compareTo(IntegerPartitionChunk.make(null, null, 0, 1)) + make(null, null, 0, 1).compareTo(make(null, null, 0, 1)) ); Assert.assertEquals( 0, - IntegerPartitionChunk.make(10, null, 0, 1).compareTo(IntegerPartitionChunk.make(10, null, 0, 2)) + make(10, null, 0, 1).compareTo(make(10, null, 0, 2)) ); Assert.assertEquals( 0, - IntegerPartitionChunk.make(null, 10, 0, 1).compareTo(IntegerPartitionChunk.make(null, 10, 0, 2)) + make(null, 10, 0, 1).compareTo(make(null, 10, 0, 2)) ); Assert.assertEquals( 0, - IntegerPartitionChunk.make(10, 11, 0, 1).compareTo(IntegerPartitionChunk.make(10, 11, 0, 2)) + make(10, 11, 0, 1).compareTo(make(10, 11, 0, 2)) ); Assert.assertEquals( -1, - IntegerPartitionChunk.make(null, 10, 0, 1).compareTo(IntegerPartitionChunk.make(10, null, 1, 2)) + make(null, 10, 0, 1).compareTo(make(10, null, 1, 2)) ); Assert.assertEquals( -1, - IntegerPartitionChunk.make(11, 20, 0, 1).compareTo(IntegerPartitionChunk.make(20, 33, 1, 1)) + make(11, 20, 0, 1).compareTo(make(20, 33, 1, 1)) ); Assert.assertEquals( 1, - IntegerPartitionChunk.make(20, 33, 1, 1).compareTo(IntegerPartitionChunk.make(11, 20, 0, 1)) + make(20, 33, 1, 1).compareTo(make(11, 20, 0, 1)) ); Assert.assertEquals( 1, - IntegerPartitionChunk.make(10, null, 1, 1).compareTo(IntegerPartitionChunk.make(null, 10, 0, 1)) + make(10, null, 1, 1).compareTo(make(null, 10, 0, 1)) ); } @Test public void testEquals() { - Assert.assertEquals(IntegerPartitionChunk.make(null, null, 0, 1), IntegerPartitionChunk.make(null, null, 0, 1)); - Assert.assertEquals(IntegerPartitionChunk.make(null, 10, 0, 1), IntegerPartitionChunk.make(null, 10, 0, 1)); - Assert.assertEquals(IntegerPartitionChunk.make(10, null, 0, 1), IntegerPartitionChunk.make(10, null, 0, 1)); - Assert.assertEquals(IntegerPartitionChunk.make(10, 11, 0, 1), IntegerPartitionChunk.make(10, 11, 0, 1)); + Assert.assertEquals(make(null, null, 0, 1), make(null, null, 0, 1)); + Assert.assertEquals(make(null, 10, 0, 1), make(null, 10, 0, 1)); + Assert.assertEquals(make(10, null, 0, 1), make(10, null, 0, 1)); + Assert.assertEquals(make(10, 11, 0, 1), make(10, 11, 0, 1)); + } + + private static class OvershadowableInteger implements Overshadowable + { + private final int val; + + OvershadowableInteger(int val) + { + this.val = val; + } + + @Override + public int getStartRootPartitionId() + { + return 0; + } + + @Override + public int getEndRootPartitionId() + { + return 1; + } + + @Override + public String getVersion() + { + return ""; + } + + @Override + public short getMinorVersion() + { + return 0; + } + + @Override + public short getAtomicUpdateGroupSize() + { + return 1; + } } } diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/IncrementalPublishingKafkaIndexTaskRunner.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/IncrementalPublishingKafkaIndexTaskRunner.java index 3b34454c8526..c77725d6ffb1 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/IncrementalPublishingKafkaIndexTaskRunner.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/IncrementalPublishingKafkaIndexTaskRunner.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; import org.apache.druid.data.input.impl.InputRowParser; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; import org.apache.druid.indexing.seekablestream.SeekableStreamDataSourceMetadata; @@ -70,7 +71,8 @@ public IncrementalPublishingKafkaIndexTaskRunner( AuthorizerMapper authorizerMapper, Optional chatHandlerProvider, CircularBuffer savedParseExceptions, - RowIngestionMetersFactory rowIngestionMetersFactory + RowIngestionMetersFactory rowIngestionMetersFactory, + LockGranularity lockGranularityToUse ) { super( @@ -79,7 +81,8 @@ public IncrementalPublishingKafkaIndexTaskRunner( authorizerMapper, chatHandlerProvider, savedParseExceptions, - rowIngestionMetersFactory + rowIngestionMetersFactory, + lockGranularityToUse ); this.task = task; } diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTask.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTask.java index b0b0c6be7470..25140d8d12f6 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTask.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTask.java @@ -135,7 +135,8 @@ protected SeekableStreamIndexTaskRunner createTaskRunner() authorizerMapper, chatHandlerProvider, savedParseExceptions, - rowIngestionMetersFactory + rowIngestionMetersFactory, + lockGranularityToUse ); } diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java index 3fd6511c7527..7bf357da2d8a 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicates; -import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -50,6 +49,7 @@ import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.IngestionStatsAndErrorsTaskReportData; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.SegmentLoaderFactory; import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.TaskReport; @@ -67,6 +67,7 @@ import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; import org.apache.druid.indexing.common.task.IndexTaskTest; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.indexing.kafka.supervisor.KafkaSupervisor; import org.apache.druid.indexing.kafka.supervisor.KafkaSupervisorIOConfig; import org.apache.druid.indexing.kafka.test.TestBroker; @@ -89,6 +90,7 @@ import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.concurrent.ListenableFutures; import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.guava.Comparators; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.apache.druid.java.util.emitter.EmittingLogger; @@ -162,6 +164,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import javax.annotation.Nullable; import java.io.File; @@ -171,6 +175,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -183,7 +188,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +@RunWith(Parameterized.class) public class KafkaIndexTaskTest { private static final Logger log = new Logger(KafkaIndexTaskTest.class); @@ -200,7 +207,17 @@ public class KafkaIndexTaskTest new KafkaIndexTaskModule().getJacksonModules().forEach(OBJECT_MAPPER::registerModule); } + @Parameterized.Parameters(name = "{0}") + public static Iterable constructorFeeder() + { + return ImmutableList.of( + new Object[]{LockGranularity.TIME_CHUNK}, + new Object[]{LockGranularity.SEGMENT} + ); + } + private final List runningTasks = new ArrayList<>(); + private final LockGranularity lockGranularity; private long handoffConditionTimeout = 0; private boolean reportParseExceptions = false; @@ -309,6 +326,11 @@ private static String getTopicName() @Rule public final TestDerbyConnector.DerbyConnectorRule derby = new TestDerbyConnector.DerbyConnectorRule(); + public KafkaIndexTaskTest(LockGranularity lockGranularity) + { + this.lockGranularity = lockGranularity; + } + @BeforeClass public static void setupClass() throws Exception { @@ -414,17 +436,18 @@ public void testRunAfterDataInserted() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -464,17 +487,18 @@ public void testRunBeforeDataInserted() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -553,14 +577,14 @@ public void testIncrementalHandOff() throws Exception Assert.assertEquals(1, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2009/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc4 = sd(task, "2011/P1D", 0); - SegmentDescriptor desc5 = sd(task, "2011/P1D", 1); - SegmentDescriptor desc6 = sd(task, "2012/P1D", 0); - SegmentDescriptor desc7 = sd(task, "2013/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + SegmentDescriptor desc3 = sd("2010/P1D", 0); + SegmentDescriptor desc4 = sd("2011/P1D", 0); + SegmentDescriptor desc5 = sd("2011/P1D", 1); + SegmentDescriptor desc6 = sd("2012/P1D", 0); + SegmentDescriptor desc7 = sd("2013/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 10L, 1, 2L)) @@ -569,15 +593,16 @@ public void testIncrementalHandOff() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc3)); - Assert.assertTrue((ImmutableList.of("d", "e").equals(readSegmentColumn("dim1", desc4)) - && ImmutableList.of("h").equals(readSegmentColumn("dim1", desc5))) || - (ImmutableList.of("d", "h").equals(readSegmentColumn("dim1", desc4)) - && ImmutableList.of("e").equals(readSegmentColumn("dim1", desc5)))); - Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", desc6)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc7)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(2))); + Assert.assertTrue((ImmutableList.of("d", "e").equals(readSegmentColumn("dim1", publishedDescriptors.get(3))) + && ImmutableList.of("h").equals(readSegmentColumn("dim1", publishedDescriptors.get(4)))) || + (ImmutableList.of("d", "h").equals(readSegmentColumn("dim1", publishedDescriptors.get(3))) + && ImmutableList.of("e").equals(readSegmentColumn("dim1", publishedDescriptors.get(4))))); + Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", publishedDescriptors.get(5))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(6))); } @Test(timeout = 60_000L) @@ -697,14 +722,14 @@ public void testIncrementalHandOffMaxTotalRows() throws Exception Assert.assertEquals(1, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2009/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc4 = sd(task, "2011/P1D", 0); - SegmentDescriptor desc5 = sd(task, "2011/P1D", 1); - SegmentDescriptor desc6 = sd(task, "2012/P1D", 0); - SegmentDescriptor desc7 = sd(task, "2013/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + SegmentDescriptor desc3 = sd("2010/P1D", 0); + SegmentDescriptor desc4 = sd("2011/P1D", 0); + SegmentDescriptor desc5 = sd("2011/P1D", 1); + SegmentDescriptor desc6 = sd("2012/P1D", 0); + SegmentDescriptor desc7 = sd("2013/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 10L, 1, 2L)) @@ -712,7 +737,7 @@ public void testIncrementalHandOffMaxTotalRows() throws Exception metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 10L, 1, 2L)) @@ -721,15 +746,16 @@ public void testIncrementalHandOffMaxTotalRows() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc3)); - Assert.assertTrue((ImmutableList.of("d", "e").equals(readSegmentColumn("dim1", desc4)) - && ImmutableList.of("h").equals(readSegmentColumn("dim1", desc5))) || - (ImmutableList.of("d", "h").equals(readSegmentColumn("dim1", desc4)) - && ImmutableList.of("e").equals(readSegmentColumn("dim1", desc5)))); - Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", desc6)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc7)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(2))); + Assert.assertTrue((ImmutableList.of("d", "e").equals(readSegmentColumn("dim1", publishedDescriptors.get(3))) + && ImmutableList.of("h").equals(readSegmentColumn("dim1", publishedDescriptors.get(4)))) || + (ImmutableList.of("d", "h").equals(readSegmentColumn("dim1", publishedDescriptors.get(3))) + && ImmutableList.of("e").equals(readSegmentColumn("dim1", publishedDescriptors.get(4))))); + Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", publishedDescriptors.get(5))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(6))); } @Test(timeout = 60_000L) @@ -804,9 +830,9 @@ public void testTimeBasedIncrementalHandOff() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2009/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 2L, 1, 0L)) @@ -815,8 +841,9 @@ public void testTimeBasedIncrementalHandOff() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -957,17 +984,18 @@ public void testRunWithMinimumMessageTime() throws Exception Assert.assertEquals(2, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + final List publishedDescriptors = publishedDescriptors(); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -1007,19 +1035,20 @@ public void testRunWithMaximumMessageTime() throws Exception Assert.assertEquals(2, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2009/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2010/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + SegmentDescriptor desc3 = sd("2010/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc3)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(2))); } @Test(timeout = 60_000L) @@ -1067,16 +1096,17 @@ public void testRunWithTransformSpec() throws Exception Assert.assertEquals(4, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2009/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2009/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("bb"), readSegmentColumn("dim1t", desc1)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("bb"), readSegmentColumn("dim1t", publishedDescriptors.get(0))); } @Test(timeout = 60_000L) @@ -1111,7 +1141,7 @@ public void testRunOnNothing() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - Assert.assertEquals(ImmutableSet.of(), publishedDescriptors()); + Assert.assertEquals(ImmutableList.of(), publishedDescriptors()); } @Test(timeout = 60_000L) @@ -1148,17 +1178,18 @@ public void testHandoffConditionTimeoutWhenHandoffOccurs() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -1196,9 +1227,10 @@ public void testHandoffConditionTimeoutWhenHandoffDoesNotOccur() throws Exceptio Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + final List publishedDescriptors = publishedDescriptors(); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors); Assert.assertEquals( new KafkaDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L)) @@ -1207,8 +1239,8 @@ public void testHandoffConditionTimeoutWhenHandoffDoesNotOccur() throws Exceptio ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -1249,7 +1281,7 @@ public void testReportParseExceptions() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - Assert.assertEquals(ImmutableSet.of(), publishedDescriptors()); + Assert.assertEquals(ImmutableList.of(), publishedDescriptors()); Assert.assertNull(metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource())); } @@ -1293,11 +1325,11 @@ public void testMultipleParseExceptionsSuccess() throws Exception Assert.assertEquals(1, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2013/P1D", 0); - SegmentDescriptor desc4 = sd(task, "2049/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + SegmentDescriptor desc3 = sd("2013/P1D", 0); + SegmentDescriptor desc4 = sd("2049/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 13L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) @@ -1371,7 +1403,7 @@ public void testMultipleParseExceptionsFailure() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - Assert.assertEquals(ImmutableSet.of(), publishedDescriptors()); + Assert.assertEquals(ImmutableList.of(), publishedDescriptors()); Assert.assertNull(metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource())); IngestionStatsAndErrorsTaskReportData reportData = getTaskReportData(); @@ -1449,17 +1481,18 @@ public void testRunReplicas() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -1514,17 +1547,18 @@ public void testRunConflicting() throws Exception Assert.assertEquals(1, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata, should all be from the first task - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + final List publishedDescriptors = publishedDescriptors(); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -1567,9 +1601,9 @@ public void testRunConflictingWithoutTransactions() throws Exception Assert.assertEquals(TaskState.SUCCESS, future1.get().getStatusCode()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertNull(metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource())); // Run second task @@ -1585,16 +1619,17 @@ public void testRunConflictingWithoutTransactions() throws Exception Assert.assertEquals(1, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc3 = sd(task2, "2011/P1D", 1); - SegmentDescriptor desc4 = sd(task2, "2013/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4), publishedDescriptors()); + SegmentDescriptor desc3 = sd("2011/P1D", 1); + SegmentDescriptor desc4 = sd("2013/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4), publishedDescriptors()); Assert.assertNull(metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource())); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc3)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc4)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(2))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(3))); } @Test(timeout = 60_000L) @@ -1629,12 +1664,13 @@ public void testRunOneTaskTwoPartitions() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); // desc3 will not be created in KafkaIndexTask (0.12.x) as it does not create per Kafka partition Druid segments - SegmentDescriptor desc3 = sd(task, "2011/P1D", 1); - SegmentDescriptor desc4 = sd(task, "2012/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc4), publishedDescriptors()); + SegmentDescriptor desc3 = sd("2011/P1D", 1); + SegmentDescriptor desc4 = sd("2012/P1D", 0); + final List publishedDescriptors = publishedDescriptors(); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc4), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L, 1, 2L)) @@ -1643,13 +1679,19 @@ public void testRunOneTaskTwoPartitions() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", desc4)); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals( + ImmutableList.of("g"), + readSegmentColumn( + "dim1", + publishedDescriptors.get(2) + ) + ); // Check desc2/desc3 without strong ordering because two partitions are interleaved nondeterministically Assert.assertEquals( ImmutableSet.of(ImmutableList.of("d", "e", "h")), - ImmutableSet.of(readSegmentColumn("dim1", desc2)) + ImmutableSet.of(readSegmentColumn("dim1", publishedDescriptors.get(1))) ); } @@ -1704,10 +1746,10 @@ public void testRunTwoTasksTwoPartitions() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - SegmentDescriptor desc3 = sd(task2, "2012/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + SegmentDescriptor desc3 = sd("2012/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L, 1, 1L)) @@ -1716,9 +1758,10 @@ public void testRunTwoTasksTwoPartitions() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", desc3)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", publishedDescriptors.get(2))); } @Test(timeout = 60_000L) @@ -1804,17 +1847,18 @@ public void testRestore() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 6L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -1913,14 +1957,14 @@ public void testRestoreAfterPersistingSequences() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2008/P1D", 1); - SegmentDescriptor desc3 = sd(task1, "2009/P1D", 0); - SegmentDescriptor desc4 = sd(task1, "2009/P1D", 1); - SegmentDescriptor desc5 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc6 = sd(task1, "2011/P1D", 0); - SegmentDescriptor desc7 = sd(task1, "2012/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2008/P1D", 1); + SegmentDescriptor desc3 = sd("2009/P1D", 0); + SegmentDescriptor desc4 = sd("2009/P1D", 1); + SegmentDescriptor desc5 = sd("2010/P1D", 0); + SegmentDescriptor desc6 = sd("2011/P1D", 0); + SegmentDescriptor desc7 = sd("2012/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 10L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) @@ -2003,17 +2047,18 @@ public void testRunWithPauseAndResume() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + final List publishedDescriptors = publishedDescriptors(); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 6L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -2126,17 +2171,18 @@ public void testRunContextSequenceAheadOfStartingOffsets() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + final List publishedDescriptors = publishedDescriptors(); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -2267,21 +2313,22 @@ public void testRunTransactionModeRollback() throws Exception Assert.assertEquals(1, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2009/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2013/P1D", 0); - SegmentDescriptor desc4 = sd(task, "2049/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + SegmentDescriptor desc3 = sd("2013/P1D", 0); + SegmentDescriptor desc4 = sd("2049/P1D", 0); + final List publishedDescriptors = publishedDescriptors(); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4), publishedDescriptors); Assert.assertEquals( new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 13L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc3)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc4)); + Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(2))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(3))); } @Test(timeout = 60_000L) @@ -2391,6 +2438,7 @@ private ListenableFuture runTask(final Task task) return taskExec.submit( () -> { try { + task.addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, lockGranularity == LockGranularity.TIME_CHUNK); if (task.isReady(toolbox.getTaskActionClient())) { return task.run(toolbox); } else { @@ -2587,7 +2635,7 @@ private void makeToolboxFactory() throws IOException derby.metadataTablesConfigSupplier().get(), derbyConnector ); - taskLockbox = new TaskLockbox(taskStorage); + taskLockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator); final TaskActionToolbox taskActionToolbox = new TaskActionToolbox( taskLockbox, taskStorage, @@ -2690,14 +2738,12 @@ private void destroyToolboxFactory() metadataStorageCoordinator = null; } - private Set publishedDescriptors() + private List publishedDescriptors() { - return FluentIterable.from( - metadataStorageCoordinator.getUsedSegmentsForInterval( - DATA_SCHEMA.getDataSource(), - Intervals.of("0000/3000") - ) - ).transform(DataSegment::toDescriptor).toSet(); + return metadataStorageCoordinator.getUsedSegmentsForInterval( + DATA_SCHEMA.getDataSource(), + Intervals.of("0000/3000") + ).stream().map(DataSegment::toDescriptor).collect(Collectors.toList()); } private void unlockAppenderatorBasePersistDirForTask(KafkaIndexTask task) @@ -2797,10 +2843,33 @@ private static byte[] jb(String timestamp, String dim1, String dim2, String dimL } } - private SegmentDescriptor sd(final Task task, final String intervalString, final int partitionNum) + private SegmentDescriptor sd(final String intervalString, final int partitionNum) { final Interval interval = Intervals.of(intervalString); - return new SegmentDescriptor(interval, getLock(task, interval).getVersion(), partitionNum); + return new SegmentDescriptor(interval, "fakeVersion", partitionNum); + } + + private void assertEqualsExceptVersion(List descriptors1, List descriptors2) + { + Assert.assertEquals(descriptors1.size(), descriptors2.size()); + final Comparator comparator = (s1, s2) -> { + final int intervalCompare = Comparators.intervalsByStartThenEnd().compare(s1.getInterval(), s2.getInterval()); + if (intervalCompare == 0) { + return Integer.compare(s1.getPartitionNumber(), s2.getPartitionNumber()); + } else { + return intervalCompare; + } + }; + + final List copy1 = new ArrayList<>(descriptors1); + final List copy2 = new ArrayList<>(descriptors2); + copy1.sort(comparator); + copy2.sort(comparator); + + for (int i = 0; i < copy1.size(); i++) { + Assert.assertEquals(copy1.get(i).getInterval(), copy2.get(i).getInterval()); + Assert.assertEquals(copy1.get(i).getPartitionNumber(), copy2.get(i).getPartitionNumber()); + } } private IngestionStatsAndErrorsTaskReportData getTaskReportData() throws IOException diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTask.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTask.java index f3dfe3bfcef0..3490ee99050e 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTask.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTask.java @@ -68,7 +68,6 @@ public KinesisIndexTask( this.awsCredentialsConfig = awsCredentialsConfig; } - @Override protected SeekableStreamIndexTaskRunner createTaskRunner() { @@ -79,7 +78,8 @@ protected SeekableStreamIndexTaskRunner createTaskRunner() authorizerMapper, chatHandlerProvider, savedParseExceptions, - rowIngestionMetersFactory + rowIngestionMetersFactory, + lockGranularityToUse ); } diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskRunner.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskRunner.java index 335119a08b19..3e88bfdb5787 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskRunner.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskRunner.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; import org.apache.druid.data.input.impl.InputRowParser; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; import org.apache.druid.indexing.seekablestream.SeekableStreamDataSourceMetadata; @@ -64,7 +65,8 @@ public class KinesisIndexTaskRunner extends SeekableStreamIndexTaskRunner chatHandlerProvider, CircularBuffer savedParseExceptions, - RowIngestionMetersFactory rowIngestionMetersFactory + RowIngestionMetersFactory rowIngestionMetersFactory, + LockGranularity lockGranularityToUse ) { super( @@ -73,7 +75,8 @@ public class KinesisIndexTaskRunner extends SeekableStreamIndexTaskRunner runningTasks = new ArrayList<>(); + @Parameterized.Parameters(name = "{0}") + public static Iterable constructorFeeder() + { + return ImmutableList.of( + new Object[]{LockGranularity.TIME_CHUNK}, + new Object[]{LockGranularity.SEGMENT} + ); + } + + private final LockGranularity lockGranularity; + private long handoffConditionTimeout = 0; private boolean reportParseExceptions = false; private boolean logParseExceptions = true; @@ -278,6 +296,11 @@ public static void setupClass() ); } + public KinesisIndexTaskTest(LockGranularity lockGranularity) + { + this.lockGranularity = lockGranularity; + } + @Before public void setupTest() throws IOException, InterruptedException { @@ -428,9 +451,9 @@ public void testRunAfterDataInserted() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4")) @@ -439,8 +462,9 @@ public void testRunAfterDataInserted() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 120_000L) @@ -497,9 +521,9 @@ public void testRunBeforeDataInserted() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2011/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2012/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2011/P1D", 0); + SegmentDescriptor desc2 = sd("2012/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID0, "1")) @@ -508,8 +532,9 @@ public void testRunBeforeDataInserted() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("h"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("h"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 120_000L) @@ -601,14 +626,14 @@ public void testIncrementalHandOff() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2009/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc4 = sd(task, "2011/P1D", 0); - SegmentDescriptor desc5 = sd(task, "2011/P1D", 1); - SegmentDescriptor desc6 = sd(task, "2012/P1D", 0); - SegmentDescriptor desc7 = sd(task, "2013/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + SegmentDescriptor desc3 = sd("2010/P1D", 0); + SegmentDescriptor desc4 = sd("2011/P1D", 0); + SegmentDescriptor desc5 = sd("2011/P1D", 1); + SegmentDescriptor desc6 = sd("2012/P1D", 0); + SegmentDescriptor desc7 = sd("2013/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>( @@ -620,15 +645,16 @@ public void testIncrementalHandOff() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc3)); - Assert.assertTrue((ImmutableList.of("d", "e").equals(readSegmentColumn("dim1", desc4)) - && ImmutableList.of("h").equals(readSegmentColumn("dim1", desc5))) || - (ImmutableList.of("d", "h").equals(readSegmentColumn("dim1", desc4)) - && ImmutableList.of("e").equals(readSegmentColumn("dim1", desc5)))); - Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", desc6)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc7)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(2))); + Assert.assertTrue((ImmutableList.of("d", "e").equals(readSegmentColumn("dim1", publishedDescriptors.get(3))) + && ImmutableList.of("h").equals(readSegmentColumn("dim1", publishedDescriptors.get(4)))) || + (ImmutableList.of("d", "h").equals(readSegmentColumn("dim1", publishedDescriptors.get(3))) + && ImmutableList.of("e").equals(readSegmentColumn("dim1", publishedDescriptors.get(4))))); + Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", publishedDescriptors.get(5))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(6))); } @Test(timeout = 120_000L) @@ -751,25 +777,26 @@ public void testIncrementalHandOffMaxTotalRows() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2009/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc4 = sd(task, "2011/P1D", 0); - SegmentDescriptor desc5 = sd(task, "2049/P1D", 0); - SegmentDescriptor desc7 = sd(task, "2013/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4, desc5, desc7), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + SegmentDescriptor desc3 = sd("2010/P1D", 0); + SegmentDescriptor desc4 = sd("2011/P1D", 0); + SegmentDescriptor desc5 = sd("2049/P1D", 0); + SegmentDescriptor desc7 = sd("2013/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4, desc5, desc7), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "10"))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc3)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc4)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc5)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc7)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(2))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(3))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(4))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(5))); } @@ -828,17 +855,18 @@ public void testRunWithMinimumMessageTime() throws Exception Assert.assertEquals(2, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4"))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @@ -897,10 +925,10 @@ public void testRunWithMaximumMessageTime() throws Exception Assert.assertEquals(2, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2009/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2010/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + SegmentDescriptor desc3 = sd("2010/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4"))), @@ -908,9 +936,10 @@ public void testRunWithMaximumMessageTime() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc3)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("a"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(2))); } @@ -977,8 +1006,8 @@ public void testRunWithTransformSpec() throws Exception Assert.assertEquals(4, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2009/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2009/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4"))), @@ -986,8 +1015,9 @@ public void testRunWithTransformSpec() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("bb"), readSegmentColumn("dim1t", desc1)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("b"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("bb"), readSegmentColumn("dim1t", publishedDescriptors.get(0))); } @@ -1043,7 +1073,7 @@ public void testRunOnSingletonRange() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - Assert.assertEquals(ImmutableSet.of(sd(task, "2010/P1D", 0)), publishedDescriptors()); + assertEqualsExceptVersion(ImmutableList.of(sd("2010/P1D", 0)), publishedDescriptors()); } @@ -1099,9 +1129,9 @@ public void testHandoffConditionTimeoutWhenHandoffOccurs() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4")) @@ -1110,8 +1140,9 @@ public void testHandoffConditionTimeoutWhenHandoffOccurs() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @@ -1168,9 +1199,9 @@ public void testHandoffConditionTimeoutWhenHandoffDoesNotOccur() throws Exceptio Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4")) @@ -1179,8 +1210,9 @@ public void testHandoffConditionTimeoutWhenHandoffDoesNotOccur() throws Exceptio ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @@ -1240,7 +1272,7 @@ public void testReportParseExceptions() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - Assert.assertEquals(ImmutableSet.of(), publishedDescriptors()); + Assert.assertEquals(ImmutableList.of(), publishedDescriptors()); Assert.assertNull(metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource())); } @@ -1303,11 +1335,11 @@ public void testMultipleParseExceptionsSuccess() throws Exception Assert.assertEquals(4, task.getRunner().getRowIngestionMeters().getUnparseable()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - SegmentDescriptor desc3 = sd(task, "2013/P1D", 0); - SegmentDescriptor desc4 = sd(task, "2049/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + SegmentDescriptor desc3 = sd("2013/P1D", 0); + SegmentDescriptor desc4 = sd("2049/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "12")) @@ -1402,7 +1434,7 @@ public void testMultipleParseExceptionsFailure() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - Assert.assertEquals(ImmutableSet.of(), publishedDescriptors()); + Assert.assertEquals(ImmutableList.of(), publishedDescriptors()); Assert.assertNull(metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource())); IngestionStatsAndErrorsTaskReportData reportData = getTaskReportData(); @@ -1504,9 +1536,9 @@ public void testRunReplicas() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4")) @@ -1515,8 +1547,9 @@ public void testRunReplicas() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @@ -1596,17 +1629,18 @@ public void testRunConflicting() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata, should all be from the first task - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4"))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @@ -1673,9 +1707,9 @@ public void testRunConflictingWithoutTransactions() throws Exception Assert.assertEquals(TaskState.SUCCESS, future1.get().getStatusCode()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertNull(metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource())); // Run second task @@ -1693,16 +1727,17 @@ public void testRunConflictingWithoutTransactions() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc3 = sd(task2, "2011/P1D", 1); - SegmentDescriptor desc4 = sd(task2, "2013/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4), publishedDescriptors()); + SegmentDescriptor desc3 = sd("2011/P1D", 1); + SegmentDescriptor desc4 = sd("2013/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4), publishedDescriptors()); Assert.assertNull(metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource())); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc3)); - Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", desc4)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(2))); + Assert.assertEquals(ImmutableList.of("f"), readSegmentColumn("dim1", publishedDescriptors.get(3))); } @@ -1765,10 +1800,10 @@ public void testRunOneTaskTwoPartitions() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - SegmentDescriptor desc4 = sd(task, "2012/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc4), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + SegmentDescriptor desc4 = sd("2012/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc4), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4", SHARD_ID0, "1")) @@ -1777,13 +1812,14 @@ public void testRunOneTaskTwoPartitions() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", desc4)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", publishedDescriptors.get(2))); // Check desc2/desc3 without strong ordering because two partitions are interleaved nondeterministically Assert.assertEquals( ImmutableSet.of(ImmutableList.of("d", "e", "h")), - ImmutableSet.of(readSegmentColumn("dim1", desc2)) + ImmutableSet.of(readSegmentColumn("dim1", publishedDescriptors.get(1))) ); } @@ -1863,12 +1899,12 @@ public void testRunTwoTasksTwoPartitions() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - SegmentDescriptor desc3 = sd(task2, "2011/P1D", 1); - SegmentDescriptor desc4 = sd(task2, "2012/P1D", 0); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + SegmentDescriptor desc3 = sd("2011/P1D", 1); + SegmentDescriptor desc4 = sd("2012/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4), publishedDescriptors()); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4", SHARD_ID0, "1")) @@ -1877,13 +1913,14 @@ public void testRunTwoTasksTwoPartitions() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); // Check desc2/desc3 without strong ordering because two partitions are interleaved nondeterministically Assert.assertEquals( ImmutableSet.of(ImmutableList.of("d", "e"), ImmutableList.of("h")), - ImmutableSet.of(readSegmentColumn("dim1", desc2), readSegmentColumn("dim1", desc3)) + ImmutableSet.of(readSegmentColumn("dim1", publishedDescriptors.get(1)), readSegmentColumn("dim1", publishedDescriptors.get(2))) ); - Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", desc4)); + Assert.assertEquals(ImmutableList.of("g"), readSegmentColumn("dim1", publishedDescriptors.get(3))); } @@ -1995,9 +2032,9 @@ public void testRestore() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "5"))), @@ -2005,8 +2042,9 @@ public void testRestore() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 120_000L) @@ -2137,13 +2175,13 @@ public void testRestoreAfterPersistingSequences() throws Exception Assert.assertEquals(0, task2.getRunner().getRowIngestionMeters().getThrownAway()); // Check published segments & metadata - SegmentDescriptor desc1 = sd(task1, "2008/P1D", 0); - SegmentDescriptor desc2 = sd(task1, "2009/P1D", 0); - SegmentDescriptor desc3 = sd(task1, "2010/P1D", 0); - SegmentDescriptor desc4 = sd(task1, "2011/P1D", 0); - SegmentDescriptor desc5 = sd(task1, "2012/P1D", 0); - SegmentDescriptor desc6 = sd(task1, "2013/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4, desc5, desc6), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2008/P1D", 0); + SegmentDescriptor desc2 = sd("2009/P1D", 0); + SegmentDescriptor desc3 = sd("2010/P1D", 0); + SegmentDescriptor desc4 = sd("2011/P1D", 0); + SegmentDescriptor desc5 = sd("2012/P1D", 0); + SegmentDescriptor desc6 = sd("2013/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2, desc3, desc4, desc5, desc6), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "6")) @@ -2239,9 +2277,9 @@ public void testRunWithPauseAndResume() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>( STREAM, @@ -2251,8 +2289,9 @@ public void testRunWithPauseAndResume() throws Exception ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 60_000L) @@ -2320,17 +2359,18 @@ public void testRunContextSequenceAheadOfStartingOffsets() throws Exception Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - SegmentDescriptor desc1 = sd(task, "2010/P1D", 0); - SegmentDescriptor desc2 = sd(task, "2011/P1D", 0); - Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + SegmentDescriptor desc1 = sd("2010/P1D", 0); + SegmentDescriptor desc2 = sd("2011/P1D", 0); + assertEqualsExceptVersion(ImmutableList.of(desc1, desc2), publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "4"))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); // Check segments in deep storage - Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); - Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + final List publishedDescriptors = publishedDescriptors(); + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", publishedDescriptors.get(0))); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", publishedDescriptors.get(1))); } @Test(timeout = 5000L) @@ -2468,16 +2508,16 @@ public void testIncrementalHandOffReadsThroughEndOffsets() throws Exception Assert.assertEquals(0, normalReplica.getRunner().getRowIngestionMeters().getThrownAway()); // Check published metadata - final Set descriptors = new HashSet<>(); - descriptors.add(sd(normalReplica, "2008/P1D", 0)); - descriptors.add(sd(normalReplica, "2009/P1D", 0)); - descriptors.add(sd(normalReplica, "2010/P1D", 0)); - descriptors.add(sd(normalReplica, "2010/P1D", 1)); - descriptors.add(sd(normalReplica, "2011/P1D", 0)); - descriptors.add(sd(normalReplica, "2011/P1D", 1)); - descriptors.add(sd(normalReplica, "2012/P1D", 0)); - descriptors.add(sd(normalReplica, "2013/P1D", 0)); - Assert.assertEquals(descriptors, publishedDescriptors()); + final List descriptors = new ArrayList<>(); + descriptors.add(sd("2008/P1D", 0)); + descriptors.add(sd("2009/P1D", 0)); + descriptors.add(sd("2010/P1D", 0)); + descriptors.add(sd("2010/P1D", 1)); + descriptors.add(sd("2011/P1D", 0)); + descriptors.add(sd("2011/P1D", 1)); + descriptors.add(sd("2012/P1D", 0)); + descriptors.add(sd("2013/P1D", 0)); + assertEqualsExceptVersion(descriptors, publishedDescriptors()); Assert.assertEquals( new KinesisDataSourceMetadata( new SeekableStreamEndSequenceNumbers<>(STREAM, ImmutableMap.of(SHARD_ID1, "9")) @@ -2571,6 +2611,7 @@ private ListenableFuture runTask(final Task task) return taskExec.submit( () -> { try { + task.addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, lockGranularity == LockGranularity.TIME_CHUNK); if (task.isReady(toolbox.getTaskActionClient())) { return task.run(toolbox); } else { @@ -2772,7 +2813,7 @@ private void makeToolboxFactory() throws IOException derby.metadataTablesConfigSupplier().get(), derbyConnector ); - taskLockbox = new TaskLockbox(taskStorage); + taskLockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator); final TaskActionToolbox taskActionToolbox = new TaskActionToolbox( taskLockbox, taskStorage, @@ -2877,14 +2918,12 @@ private void destroyToolboxFactory() } - private Set publishedDescriptors() + private List publishedDescriptors() { - return FluentIterable.from( - metadataStorageCoordinator.getUsedSegmentsForInterval( - DATA_SCHEMA.getDataSource(), - Intervals.of("0000/3000") - ) - ).transform(DataSegment::toDescriptor).toSet(); + return metadataStorageCoordinator.getUsedSegmentsForInterval( + DATA_SCHEMA.getDataSource(), + Intervals.of("0000/3000") + ).stream().map(DataSegment::toDescriptor).collect(Collectors.toList()); } private void unlockAppenderatorBasePersistDirForTask(KinesisIndexTask task) @@ -2992,10 +3031,33 @@ private static List jb( } } - private SegmentDescriptor sd(final Task task, final String intervalString, final int partitionNum) + private SegmentDescriptor sd(final String intervalString, final int partitionNum) { final Interval interval = Intervals.of(intervalString); - return new SegmentDescriptor(interval, getLock(task, interval).getVersion(), partitionNum); + return new SegmentDescriptor(interval, "fakeVersion", partitionNum); + } + + private void assertEqualsExceptVersion(List descriptors1, List descriptors2) + { + Assert.assertEquals(descriptors1.size(), descriptors2.size()); + final Comparator comparator = (s1, s2) -> { + final int intervalCompare = Comparators.intervalsByStartThenEnd().compare(s1.getInterval(), s2.getInterval()); + if (intervalCompare == 0) { + return Integer.compare(s1.getPartitionNumber(), s2.getPartitionNumber()); + } else { + return intervalCompare; + } + }; + + final List copy1 = new ArrayList<>(descriptors1); + final List copy2 = new ArrayList<>(descriptors2); + copy1.sort(comparator); + copy2.sort(comparator); + + for (int i = 0; i < copy1.size(); i++) { + Assert.assertEquals(copy1.get(i).getInterval(), copy2.get(i).getInterval()); + Assert.assertEquals(copy1.get(i).getPartitionNumber(), copy2.get(i).getPartitionNumber()); + } } private IngestionStatsAndErrorsTaskReportData getTaskReportData() throws IOException diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java index f4cad315270a..79e6d836dc42 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java @@ -823,6 +823,7 @@ public void doRun() -1, -1 ); + final DataSegment segment = JobHelper.serializeOutIndex( segmentTemplate, context.getConfiguration(), diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/LockGranularity.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/LockGranularity.java new file mode 100644 index 000000000000..37cdecc91320 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/LockGranularity.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common; + +/** + * Granularity for {@link TaskLock}. + */ +public enum LockGranularity +{ + TIME_CHUNK, + SEGMENT +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/SegmentLock.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/SegmentLock.java new file mode 100644 index 000000000000..ec8b24e8b828 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/SegmentLock.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.indexing.overlord.LockRequest; +import org.apache.druid.indexing.overlord.LockRequestForNewSegment; +import org.apache.druid.indexing.overlord.SpecificSegmentLockRequest; +import org.apache.druid.indexing.overlord.TimeChunkLockRequest; +import org.apache.druid.java.util.common.ISE; +import org.joda.time.Interval; + +import java.util.Objects; + +/** + * Lock for set of segments. Should be unique for (dataSource, interval, version, partitionId). + */ +public class SegmentLock implements TaskLock +{ + static final String TYPE = "segment"; + + private final TaskLockType lockType; + private final String groupId; + private final String dataSource; + private final Interval interval; + private final String version; + private final int partitionId; + private final int priority; + private final boolean revoked; + + @JsonCreator + public SegmentLock( + @JsonProperty("type") TaskLockType lockType, + @JsonProperty("groupId") String groupId, + @JsonProperty("dataSource") String dataSource, + @JsonProperty("interval") Interval interval, + @JsonProperty("version") String version, + @JsonProperty("partitionId") int partitionId, + @JsonProperty("priority") int priority, + @JsonProperty("revoked") boolean revoked + ) + { + this.lockType = Preconditions.checkNotNull(lockType, "lockType"); + this.groupId = Preconditions.checkNotNull(groupId, "groupId"); + this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); + this.interval = Preconditions.checkNotNull(interval, "interval"); + this.version = Preconditions.checkNotNull(version, "version"); + this.partitionId = partitionId; + this.priority = priority; + this.revoked = revoked; + } + + public SegmentLock( + TaskLockType lockType, + String groupId, + String dataSource, + Interval interval, + String version, + int partitionId, + int priority + ) + { + this(lockType, groupId, dataSource, interval, version, partitionId, priority, false); + } + + @Override + public TaskLock revokedCopy() + { + return new SegmentLock(lockType, groupId, dataSource, interval, version, partitionId, priority, true); + } + + @Override + public TaskLock withPriority(int newPriority) + { + return new SegmentLock(lockType, groupId, dataSource, interval, version, partitionId, newPriority, revoked); + } + + @Override + public LockGranularity getGranularity() + { + return LockGranularity.SEGMENT; + } + + @JsonProperty + @Override + public TaskLockType getType() + { + return lockType; + } + + @JsonProperty + @Override + public String getGroupId() + { + return groupId; + } + + @JsonProperty + @Override + public String getDataSource() + { + return dataSource; + } + + @JsonProperty + @Override + public Interval getInterval() + { + return interval; + } + + @JsonProperty + public int getPartitionId() + { + return partitionId; + } + + @JsonProperty + @Override + public String getVersion() + { + return version; + } + + @JsonProperty + @Override + public Integer getPriority() + { + return priority; + } + + @Override + public int getNonNullPriority() + { + return priority; + } + + @JsonProperty + @Override + public boolean isRevoked() + { + return revoked; + } + + @Override + public boolean conflict(LockRequest request) + { + if (request instanceof LockRequestForNewSegment) { + // request for new segments doens't conflict with any locks because it allocates a new partitionId + return false; + } + + if (!dataSource.equals(request.getDataSource())) { + return false; + } + + if (request instanceof TimeChunkLockRequest) { + // For different interval, all overlapping intervals cause conflict. + return interval.overlaps(request.getInterval()); + } else if (request instanceof SpecificSegmentLockRequest) { + if (interval.equals(request.getInterval())) { + final SpecificSegmentLockRequest specificSegmentLockRequest = (SpecificSegmentLockRequest) request; + // Lock conflicts only if the interval is same and the partitionIds intersect. + return specificSegmentLockRequest.getPartitionId() == partitionId; + } else { + // For different interval, all overlapping intervals cause conflict. + return interval.overlaps(request.getInterval()); + } + } else { + throw new ISE("Unknown request type[%s]", request.getClass().getName()); + } + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SegmentLock that = (SegmentLock) o; + return partitionId == that.partitionId && + priority == that.priority && + revoked == that.revoked && + lockType == that.lockType && + Objects.equals(groupId, that.groupId) && + Objects.equals(dataSource, that.dataSource) && + Objects.equals(interval, that.interval) && + Objects.equals(version, that.version); + } + + @Override + public int hashCode() + { + return Objects.hash(lockType, groupId, dataSource, interval, partitionId, version, priority, revoked); + } + + @Override + public String toString() + { + return "SegmentLock{" + + "lockType=" + lockType + + ", groupId='" + groupId + '\'' + + ", dataSource='" + dataSource + '\'' + + ", interval=" + interval + + ", version='" + version + '\'' + + ", partitionId=" + partitionId + + ", priority=" + priority + + ", revoked=" + revoked + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/TaskLock.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/TaskLock.java index 865b053b95c4..8c1e5d256033 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/TaskLock.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/TaskLock.java @@ -19,10 +19,11 @@ package org.apache.druid.indexing.common; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +import org.apache.druid.indexing.overlord.LockRequest; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -30,156 +31,35 @@ /** * Represents a lock held by some task. Immutable. */ -public class TaskLock +@JsonTypeInfo(use = Id.NAME, property = "granularity", defaultImpl = TimeChunkLock.class) +@JsonSubTypes(value = { + @Type(name = TimeChunkLock.TYPE, value = TimeChunkLock.class), + @Type(name = SegmentLock.TYPE, value = SegmentLock.class) +}) +public interface TaskLock { - private final TaskLockType type; - private final String groupId; - private final String dataSource; - private final Interval interval; - private final String version; - private final Integer priority; - private final boolean revoked; + TaskLock revokedCopy(); - public static TaskLock withPriority(TaskLock lock, int priority) - { - return new TaskLock( - lock.type, - lock.getGroupId(), - lock.getDataSource(), - lock.getInterval(), - lock.getVersion(), - priority, - lock.isRevoked() - ); - } + TaskLock withPriority(int priority); - @JsonCreator - public TaskLock( - @JsonProperty("type") @Nullable TaskLockType type, // nullable for backward compatibility - @JsonProperty("groupId") String groupId, - @JsonProperty("dataSource") String dataSource, - @JsonProperty("interval") Interval interval, - @JsonProperty("version") String version, - @JsonProperty("priority") @Nullable Integer priority, - @JsonProperty("revoked") boolean revoked - ) - { - this.type = type == null ? TaskLockType.EXCLUSIVE : type; - this.groupId = Preconditions.checkNotNull(groupId, "groupId"); - this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); - this.interval = Preconditions.checkNotNull(interval, "interval"); - this.version = Preconditions.checkNotNull(version, "version"); - this.priority = priority; - this.revoked = revoked; - } + LockGranularity getGranularity(); - public TaskLock( - TaskLockType type, - String groupId, - String dataSource, - Interval interval, - String version, - int priority - ) - { - this(type, groupId, dataSource, interval, version, priority, false); - } + TaskLockType getType(); - public TaskLock revokedCopy() - { - return new TaskLock( - type, - groupId, - dataSource, - interval, - version, - priority, - true - ); - } + String getGroupId(); - @JsonProperty - public TaskLockType getType() - { - return type; - } + String getDataSource(); - @JsonProperty - public String getGroupId() - { - return groupId; - } + Interval getInterval(); - @JsonProperty - public String getDataSource() - { - return dataSource; - } + String getVersion(); - @JsonProperty - public Interval getInterval() - { - return interval; - } - - @JsonProperty - public String getVersion() - { - return version; - } - - @JsonProperty @Nullable - public Integer getPriority() - { - return priority; - } - - public int getNonNullPriority() - { - return Preconditions.checkNotNull(priority, "priority"); - } - - @JsonProperty - public boolean isRevoked() - { - return revoked; - } + Integer getPriority(); - @Override - public boolean equals(Object o) - { - if (!(o instanceof TaskLock)) { - return false; - } else { - final TaskLock that = (TaskLock) o; - return this.type.equals(that.type) && - this.groupId.equals(that.groupId) && - this.dataSource.equals(that.dataSource) && - this.interval.equals(that.interval) && - this.version.equals(that.version) && - Objects.equal(this.priority, that.priority) && - this.revoked == that.revoked; - } - } + int getNonNullPriority(); - @Override - public int hashCode() - { - return Objects.hashCode(type, groupId, dataSource, interval, version, priority, revoked); - } + boolean isRevoked(); - @Override - public String toString() - { - return Objects.toStringHelper(this) - .add("type", type) - .add("groupId", groupId) - .add("dataSource", dataSource) - .add("interval", interval) - .add("version", version) - .add("priority", priority) - .add("revoked", revoked) - .toString(); - } + boolean conflict(LockRequest request); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/TaskLockType.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/TaskLockType.java index e1f3f4e062dc..b51990cee03e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/TaskLockType.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/TaskLockType.java @@ -22,5 +22,5 @@ public enum TaskLockType { SHARED, - EXCLUSIVE + EXCLUSIVE // taskLocks of this type can be shared by tasks of the same groupId. } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/TimeChunkLock.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/TimeChunkLock.java new file mode 100644 index 000000000000..15bb3ffceb2d --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/TimeChunkLock.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.druid.indexing.overlord.LockRequest; +import org.joda.time.Interval; + +import javax.annotation.Nullable; +import java.util.Objects; + +public class TimeChunkLock implements TaskLock +{ + static final String TYPE = "timeChunk"; + + private final TaskLockType lockType; + private final String groupId; + private final String dataSource; + private final Interval interval; + private final String version; + @Nullable + private final Integer priority; + private final boolean revoked; + + @JsonCreator + public TimeChunkLock( + @JsonProperty("type") @Nullable TaskLockType lockType, // nullable for backward compatibility + @JsonProperty("groupId") String groupId, + @JsonProperty("dataSource") String dataSource, + @JsonProperty("interval") Interval interval, + @JsonProperty("version") String version, + @JsonProperty("priority") @Nullable Integer priority, + @JsonProperty("revoked") boolean revoked + ) + { + this.lockType = lockType == null ? TaskLockType.EXCLUSIVE : lockType; + this.groupId = Preconditions.checkNotNull(groupId, "groupId"); + this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); + this.interval = Preconditions.checkNotNull(interval, "interval"); + this.version = Preconditions.checkNotNull(version, "version"); + this.priority = priority; + this.revoked = revoked; + } + + @VisibleForTesting + public TimeChunkLock( + TaskLockType type, + String groupId, + String dataSource, + Interval interval, + String version, + int priority + ) + { + this(type, groupId, dataSource, interval, version, priority, false); + } + + @Override + public TaskLock revokedCopy() + { + return new TimeChunkLock( + lockType, + groupId, + dataSource, + interval, + version, + priority, + true + ); + } + + @Override + public TaskLock withPriority(int priority) + { + return new TimeChunkLock( + this.lockType, + this.groupId, + this.dataSource, + this.interval, + this.version, + priority, + this.revoked + ); + } + + @Override + public LockGranularity getGranularity() + { + return LockGranularity.TIME_CHUNK; + } + + @Override + @JsonProperty + public TaskLockType getType() + { + return lockType; + } + + @Override + @JsonProperty + public String getGroupId() + { + return groupId; + } + + @Override + @JsonProperty + public String getDataSource() + { + return dataSource; + } + + @Override + @JsonProperty + public Interval getInterval() + { + return interval; + } + + @Override + @JsonProperty + public String getVersion() + { + return version; + } + + @Override + @JsonProperty + @Nullable + public Integer getPriority() + { + return priority; + } + + @Override + public int getNonNullPriority() + { + return Preconditions.checkNotNull(priority, "priority"); + } + + @Override + @JsonProperty + public boolean isRevoked() + { + return revoked; + } + + @Override + public boolean conflict(LockRequest request) + { + return dataSource.equals(request.getDataSource()) + && interval.overlaps(request.getInterval()); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TimeChunkLock that = (TimeChunkLock) o; + return revoked == that.revoked && + lockType == that.lockType && + Objects.equals(groupId, that.groupId) && + Objects.equals(dataSource, that.dataSource) && + Objects.equals(interval, that.interval) && + Objects.equals(version, that.version) && + Objects.equals(priority, that.priority); + } + + @Override + public int hashCode() + { + return Objects.hash(lockType, groupId, dataSource, interval, version, priority, revoked); + } + + @Override + public String toString() + { + return "TimeChunkLock{" + + "type=" + lockType + + ", groupId='" + groupId + '\'' + + ", dataSource='" + dataSource + '\'' + + ", interval=" + interval + + ", version='" + version + '\'' + + ", priority=" + priority + + ", revoked=" + revoked + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java index 97959d7be13a..9b2a7659dfb0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java @@ -19,6 +19,7 @@ package org.apache.druid.indexing.common.actions; +import com.google.common.annotations.VisibleForTesting; import org.apache.druid.indexing.common.task.IndexTaskUtils; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.TaskStorage; @@ -26,15 +27,20 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + public class LocalTaskActionClient implements TaskActionClient { + private static final EmittingLogger log = new EmittingLogger(LocalTaskActionClient.class); + + private final ConcurrentHashMap, AtomicInteger> actionCountMap = new ConcurrentHashMap<>(); + private final Task task; private final TaskStorage storage; private final TaskActionToolbox toolbox; private final TaskAuditLogConfig auditLogConfig; - private static final EmittingLogger log = new EmittingLogger(LocalTaskActionClient.class); - public LocalTaskActionClient( Task task, TaskStorage storage, @@ -73,9 +79,17 @@ public RetType submit(TaskAction taskAction) final long performStartTime = System.currentTimeMillis(); final RetType result = taskAction.perform(task, toolbox); emitTimerMetric("task/action/run/time", System.currentTimeMillis() - performStartTime); + actionCountMap.computeIfAbsent(taskAction.getClass(), k -> new AtomicInteger()).incrementAndGet(); return result; } + @VisibleForTesting + public int getActionCount(Class actionClass) + { + final AtomicInteger count = actionCountMap.get(actionClass); + return count == null ? 0 : count.get(); + } + private void emitTimerMetric(final String metric, final long time) { final ServiceMetricEvent.Builder metricBuilder = ServiceMetricEvent.builder(); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocateAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocateAction.java index 6623483667e4..aecfcec36432 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocateAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocateAction.java @@ -19,15 +19,16 @@ package org.apache.druid.indexing.common.actions; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.task.Task; -import org.apache.druid.indexing.overlord.CriticalAction; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; +import org.apache.druid.indexing.overlord.LockRequestForNewSegment; import org.apache.druid.indexing.overlord.LockResult; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; @@ -36,9 +37,13 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.NumberedShardSpecFactory; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.joda.time.DateTime; import org.joda.time.Interval; +import javax.annotation.Nullable; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; @@ -49,10 +54,12 @@ * segments for the given timestamp, or if the prior segments for the given timestamp are already at the * preferredSegmentGranularity. Otherwise, the prior segments will take precedence. *

- * This action implicitly acquires locks when it allocates segments. You do not have to acquire them beforehand, - * although you *do* have to release them yourself. + * This action implicitly acquires some task locks when it allocates segments. You do not have to acquire them + * beforehand, although you *do* have to release them yourself. (Note that task locks are automatically released when + * the task is finished.) *

- * If this action cannot acquire an appropriate lock, or if it cannot expand an existing segment set, it returns null. + * If this action cannot acquire an appropriate task lock, or if it cannot expand an existing segment set, it returns + * null. */ public class SegmentAllocateAction implements TaskAction { @@ -68,7 +75,10 @@ public class SegmentAllocateAction implements TaskAction private final String sequenceName; private final String previousSegmentId; private final boolean skipSegmentLineageCheck; + private final ShardSpecFactory shardSpecFactory; + private final LockGranularity lockGranularity; + @JsonCreator public SegmentAllocateAction( @JsonProperty("dataSource") String dataSource, @JsonProperty("timestamp") DateTime timestamp, @@ -76,7 +86,9 @@ public SegmentAllocateAction( @JsonProperty("preferredSegmentGranularity") Granularity preferredSegmentGranularity, @JsonProperty("sequenceName") String sequenceName, @JsonProperty("previousSegmentId") String previousSegmentId, - @JsonProperty("skipSegmentLineageCheck") boolean skipSegmentLineageCheck + @JsonProperty("skipSegmentLineageCheck") boolean skipSegmentLineageCheck, + @JsonProperty("shardSpecFactory") @Nullable ShardSpecFactory shardSpecFactory, // nullable for backward compatibility + @JsonProperty("lockGranularity") @Nullable LockGranularity lockGranularity // nullable for backward compatibility ) { this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); @@ -89,6 +101,8 @@ public SegmentAllocateAction( this.sequenceName = Preconditions.checkNotNull(sequenceName, "sequenceName"); this.previousSegmentId = previousSegmentId; this.skipSegmentLineageCheck = skipSegmentLineageCheck; + this.shardSpecFactory = shardSpecFactory == null ? NumberedShardSpecFactory.instance() : shardSpecFactory; + this.lockGranularity = lockGranularity == null ? LockGranularity.TIME_CHUNK : lockGranularity; } @JsonProperty @@ -133,6 +147,18 @@ public boolean isSkipSegmentLineageCheck() return skipSegmentLineageCheck; } + @JsonProperty + public ShardSpecFactory getShardSpecFactory() + { + return shardSpecFactory; + } + + @JsonProperty + public LockGranularity getLockGranularity() + { + return lockGranularity; + } + @Override public TypeReference getReturnTypeReference() { @@ -162,18 +188,18 @@ public SegmentIdWithShardSpec perform( final Interval rowInterval = queryGranularity.bucket(timestamp); - final Set usedSegmentsForRow = ImmutableSet.copyOf( + final Set usedSegmentsForRow = new HashSet<>( msc.getUsedSegmentsForInterval(dataSource, rowInterval) ); final SegmentIdWithShardSpec identifier = usedSegmentsForRow.isEmpty() ? tryAllocateFirstSegment(toolbox, task, rowInterval) : tryAllocateSubsequentSegment( - toolbox, - task, - rowInterval, - usedSegmentsForRow.iterator().next() - ); + toolbox, + task, + rowInterval, + usedSegmentsForRow.iterator().next() + ); if (identifier != null) { return identifier; } @@ -257,43 +283,31 @@ private SegmentIdWithShardSpec tryAllocate( boolean logOnFail ) { - log.debug( - "Trying to allocate pending segment for rowInterval[%s], segmentInterval[%s].", - rowInterval, - tryInterval + // This action is always used by appending tasks, which cannot change the segment granularity of existing + // dataSources. So, all lock requests should be segmentLock. + final LockResult lockResult = toolbox.getTaskLockbox().tryLock( + task, + new LockRequestForNewSegment( + lockGranularity, + TaskLockType.EXCLUSIVE, + task.getGroupId(), + dataSource, + tryInterval, + shardSpecFactory, + task.getPriority(), + sequenceName, + previousSegmentId, + skipSegmentLineageCheck + ) ); - final LockResult lockResult = toolbox.getTaskLockbox().tryLock(TaskLockType.EXCLUSIVE, task, tryInterval); + if (lockResult.isRevoked()) { // We had acquired a lock but it was preempted by other locks throw new ISE("The lock for interval[%s] is preempted and no longer valid", tryInterval); } if (lockResult.isOk()) { - final SegmentIdWithShardSpec identifier; - try { - identifier = toolbox.getTaskLockbox().doInCriticalSection( - task, - ImmutableList.of(tryInterval), - CriticalAction - .builder() - .onValidLocks( - () -> toolbox.getIndexerMetadataStorageCoordinator().allocatePendingSegment( - dataSource, - sequenceName, - previousSegmentId, - tryInterval, - lockResult.getTaskLock().getVersion(), - skipSegmentLineageCheck - ) - ) - .onInvalidLocks(() -> null) - .build() - ); - } - catch (Exception e) { - throw new RuntimeException(e); - } - + final SegmentIdWithShardSpec identifier = lockResult.getNewSegmentId(); if (identifier != null) { return identifier; } else { @@ -340,7 +354,9 @@ public String toString() ", preferredSegmentGranularity=" + preferredSegmentGranularity + ", sequenceName='" + sequenceName + '\'' + ", previousSegmentId='" + previousSegmentId + '\'' + - ", skipSegmentLineageCheck='" + skipSegmentLineageCheck + '\'' + + ", skipSegmentLineageCheck=" + skipSegmentLineageCheck + + ", shardSpecFactory=" + shardSpecFactory + + ", lockGranularity=" + lockGranularity + '}'; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentInsertAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentInsertAction.java index f424c7ce6f02..e7804564ef8a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentInsertAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentInsertAction.java @@ -71,7 +71,7 @@ public TypeReference> getReturnTypeReference() @Override public Set perform(Task task, TaskActionToolbox toolbox) { - return new SegmentTransactionalInsertAction(segments, null, null).perform(task, toolbox).getSegments(); + return SegmentTransactionalInsertAction.appendAction(segments, null, null).perform(task, toolbox).getSegments(); } @Override diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentListUsedAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentListUsedAction.java index 50c22d33d2af..461db0ae624f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentListUsedAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentListUsedAction.java @@ -30,6 +30,7 @@ import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; +import java.util.Collection; import java.util.List; public class SegmentListUsedAction implements TaskAction> @@ -44,7 +45,7 @@ public class SegmentListUsedAction implements TaskAction> public SegmentListUsedAction( @JsonProperty("dataSource") String dataSource, @Deprecated @JsonProperty("interval") Interval interval, - @JsonProperty("intervals") List intervals + @JsonProperty("intervals") Collection intervals ) { this.dataSource = dataSource; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockAcquireAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockAcquireAction.java new file mode 100644 index 000000000000..70c81225b2f5 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockAcquireAction.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.actions; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Preconditions; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.LockResult; +import org.apache.druid.indexing.overlord.SpecificSegmentLockRequest; +import org.joda.time.Interval; + +/** + * TaskAction to acquire a {@link org.apache.druid.indexing.common.SegmentLock}. + * This action is a blocking operation and the caller could wait until it gets {@link LockResult} + * (up to timeoutMs if it's > 0). + * + * This action is currently used by only stream ingestion tasks. + */ +public class SegmentLockAcquireAction implements TaskAction +{ + private final TaskLockType lockType; + private final Interval interval; + private final String version; + private final int partitionId; + private final long timeoutMs; + + @JsonCreator + public SegmentLockAcquireAction( + @JsonProperty("lockType") TaskLockType lockType, + @JsonProperty("interval") Interval interval, + @JsonProperty("version") String version, + @JsonProperty("partitionId") int partitionId, + @JsonProperty("timeoutMs") long timeoutMs + ) + { + this.lockType = Preconditions.checkNotNull(lockType, "lockType"); + this.interval = Preconditions.checkNotNull(interval, "interval"); + this.version = Preconditions.checkNotNull(version, "version"); + this.partitionId = partitionId; + this.timeoutMs = timeoutMs; + } + + @JsonProperty + public TaskLockType getLockType() + { + return lockType; + } + + @JsonProperty + public Interval getInterval() + { + return interval; + } + + @JsonProperty + public String getVersion() + { + return version; + } + + @JsonProperty + public int getPartitionId() + { + return partitionId; + } + + @JsonProperty + public long getTimeoutMs() + { + return timeoutMs; + } + + @Override + public TypeReference getReturnTypeReference() + { + return new TypeReference() + { + }; + } + + @Override + public LockResult perform(Task task, TaskActionToolbox toolbox) + { + try { + if (timeoutMs == 0) { + return toolbox.getTaskLockbox().lock( + task, + new SpecificSegmentLockRequest(lockType, task, interval, version, partitionId) + ); + } else { + return toolbox.getTaskLockbox().lock( + task, + new SpecificSegmentLockRequest(lockType, task, interval, version, partitionId), timeoutMs + ); + } + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean isAudited() + { + return false; + } + + @Override + public String toString() + { + return "SegmentLockAcquireAction{" + + "lockType=" + lockType + + ", interval=" + interval + + ", version='" + version + '\'' + + ", partitionId=" + partitionId + + ", timeoutMs=" + timeoutMs + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockReleaseAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockReleaseAction.java new file mode 100644 index 000000000000..6a47091710bd --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockReleaseAction.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.actions; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.druid.indexing.common.task.Task; +import org.joda.time.Interval; + +/** + * TaskAction to release a {@link org.apache.druid.indexing.common.SegmentLock}. + * Used by batch tasks when they fail to acquire all necessary locks. + */ +public class SegmentLockReleaseAction implements TaskAction +{ + private final Interval interval; + private final int partitionId; + + @JsonCreator + public SegmentLockReleaseAction(@JsonProperty Interval interval, @JsonProperty int partitionId) + { + this.interval = interval; + this.partitionId = partitionId; + } + + @JsonProperty + public Interval getInterval() + { + return interval; + } + + @JsonProperty + public int getPartitionId() + { + return partitionId; + } + + @Override + public TypeReference getReturnTypeReference() + { + return new TypeReference() + { + }; + } + + @Override + public Void perform(Task task, TaskActionToolbox toolbox) + { + toolbox.getTaskLockbox().unlock(task, interval, partitionId); + return null; + } + + @Override + public boolean isAudited() + { + return false; + } + + @Override + public String toString() + { + return "SegmentLockReleaseAction{" + + "interval=" + interval + + ", partitionId=" + partitionId + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockTryAcquireAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockTryAcquireAction.java new file mode 100644 index 000000000000..7728574c0756 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentLockTryAcquireAction.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.actions; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Preconditions; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.LockResult; +import org.apache.druid.indexing.overlord.SpecificSegmentLockRequest; +import org.joda.time.Interval; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * TaskAction to try to acquire a {@link org.apache.druid.indexing.common.SegmentLock}. + * This action returns immediately failed {@link LockResult} if it fails to get locks for the given partitionIds. + */ +public class SegmentLockTryAcquireAction implements TaskAction> +{ + private final TaskLockType type; + private final Interval interval; + private final String version; + private final Set partitionIds; + + @JsonCreator + public SegmentLockTryAcquireAction( + @JsonProperty("lockType") TaskLockType type, + @JsonProperty("interval") Interval interval, + @JsonProperty("version") String version, + @JsonProperty("partitionIds") Set partitionIds + ) + { + Preconditions.checkState(partitionIds != null && !partitionIds.isEmpty(), "partitionIds is empty"); + this.type = Preconditions.checkNotNull(type, "type"); + this.interval = Preconditions.checkNotNull(interval, "interval"); + this.version = Preconditions.checkNotNull(version, "version"); + this.partitionIds = partitionIds; + } + + @JsonProperty("lockType") + public TaskLockType getType() + { + return type; + } + + @JsonProperty + public Interval getInterval() + { + return interval; + } + + @JsonProperty + public String getVersion() + { + return version; + } + + @JsonProperty + public Set getPartitionIds() + { + return partitionIds; + } + + @Override + public TypeReference> getReturnTypeReference() + { + return new TypeReference>() + { + }; + } + + @Override + public List perform(Task task, TaskActionToolbox toolbox) + { + return partitionIds.stream() + .map(partitionId -> toolbox.getTaskLockbox().tryLock( + task, + new SpecificSegmentLockRequest(type, task, interval, version, partitionId) + )) + .collect(Collectors.toList()); + } + + @Override + public boolean isAudited() + { + return false; + } + + @Override + public String toString() + { + return "SegmentLockTryAcquireAction{" + + "type=" + type + + ", interval=" + interval + + ", version='" + version + '\'' + + ", partitionIds=" + partitionIds + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java index 446ee1110038..83e8bb96bb23 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java @@ -32,9 +32,7 @@ import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DruidMetrics; import org.apache.druid.timeline.DataSegment; -import org.joda.time.Interval; -import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -66,14 +64,12 @@ public TypeReference getReturnTypeReference() @Override public Void perform(Task task, TaskActionToolbox toolbox) { - TaskActionPreconditions.checkLockCoversSegments(task, toolbox.getTaskLockbox(), segments); - - final List intervals = segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()); + TaskLocks.checkLockCoversSegments(task, toolbox.getTaskLockbox(), segments); try { toolbox.getTaskLockbox().doInCriticalSection( task, - intervals, + segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()), CriticalAction.builder() .onValidLocks( () -> { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java index f81028e1aad2..4197a2167d3f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java @@ -32,9 +32,7 @@ import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DruidMetrics; import org.apache.druid.timeline.DataSegment; -import org.joda.time.Interval; -import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -68,14 +66,12 @@ public TypeReference getReturnTypeReference() @Override public Void perform(Task task, TaskActionToolbox toolbox) { - TaskActionPreconditions.checkLockCoversSegments(task, toolbox.getTaskLockbox(), segments); - - final List intervals = segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()); + TaskLocks.checkLockCoversSegments(task, toolbox.getTaskLockbox(), segments); try { toolbox.getTaskLockbox().doInCriticalSection( task, - intervals, + segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()), CriticalAction.builder() .onValidLocks( () -> { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java index ca595a0505a1..e1428c6c9618 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java @@ -22,16 +22,28 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.task.IndexTaskUtils; +import org.apache.druid.indexing.common.task.SegmentLockHelper; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.CriticalAction; import org.apache.druid.indexing.overlord.DataSourceMetadata; import org.apache.druid.indexing.overlord.SegmentPublishResult; +import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DruidMetrics; import org.apache.druid.timeline.DataSegment; +import org.joda.time.Interval; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -45,30 +57,52 @@ */ public class SegmentTransactionalInsertAction implements TaskAction { - + @Nullable + private final Set segmentsToBeOverwritten; private final Set segments; + @Nullable private final DataSourceMetadata startMetadata; + @Nullable private final DataSourceMetadata endMetadata; - public SegmentTransactionalInsertAction( - Set segments + public static SegmentTransactionalInsertAction overwriteAction( + @Nullable Set segmentsToBeOverwritten, + Set segmentsToPublish + ) + { + return new SegmentTransactionalInsertAction(segmentsToBeOverwritten, segmentsToPublish, null, null); + } + + public static SegmentTransactionalInsertAction appendAction( + Set segments, + @Nullable DataSourceMetadata startMetadata, + @Nullable DataSourceMetadata endMetadata ) { - this(segments, null, null); + return new SegmentTransactionalInsertAction(null, segments, startMetadata, endMetadata); } @JsonCreator - public SegmentTransactionalInsertAction( + private SegmentTransactionalInsertAction( + @JsonProperty("segmentsToBeOverwritten") @Nullable Set segmentsToBeOverwritten, @JsonProperty("segments") Set segments, - @JsonProperty("startMetadata") DataSourceMetadata startMetadata, - @JsonProperty("endMetadata") DataSourceMetadata endMetadata + @JsonProperty("startMetadata") @Nullable DataSourceMetadata startMetadata, + @JsonProperty("endMetadata") @Nullable DataSourceMetadata endMetadata ) { + this.segmentsToBeOverwritten = segmentsToBeOverwritten; this.segments = ImmutableSet.copyOf(segments); this.startMetadata = startMetadata; this.endMetadata = endMetadata; } + @JsonProperty + @Nullable + public Set getSegmentsToBeOverwritten() + { + return segmentsToBeOverwritten; + } + @JsonProperty public Set getSegments() { @@ -76,12 +110,14 @@ public Set getSegments() } @JsonProperty + @Nullable public DataSourceMetadata getStartMetadata() { return startMetadata; } @JsonProperty + @Nullable public DataSourceMetadata getEndMetadata() { return endMetadata; @@ -96,19 +132,30 @@ public TypeReference getReturnTypeReference() } /** - * Behaves similarly to - * {@link org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator#announceHistoricalSegments(Set, DataSourceMetadata, DataSourceMetadata)}. + * Performs some sanity checks and publishes the given segments. */ @Override public SegmentPublishResult perform(Task task, TaskActionToolbox toolbox) { - TaskActionPreconditions.checkLockCoversSegments(task, toolbox.getTaskLockbox(), segments); + final Set allSegments = new HashSet<>(segments); + if (segmentsToBeOverwritten != null) { + allSegments.addAll(segmentsToBeOverwritten); + } + TaskLocks.checkLockCoversSegments(task, toolbox.getTaskLockbox(), allSegments); + + if (segmentsToBeOverwritten != null && !segmentsToBeOverwritten.isEmpty()) { + final List locks = toolbox.getTaskLockbox().findLocksForTask(task); + // Let's do some sanity check that newSegments can overwrite oldSegments. + if (locks.get(0).getGranularity() == LockGranularity.SEGMENT) { + checkWithSegmentLock(); + } + } final SegmentPublishResult retVal; try { retVal = toolbox.getTaskLockbox().doInCriticalSection( task, - segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()), + allSegments.stream().map(DataSegment::getInterval).collect(Collectors.toList()), CriticalAction.builder() .onValidLocks( () -> toolbox.getIndexerMetadataStorageCoordinator().announceHistoricalSegments( @@ -149,6 +196,67 @@ public SegmentPublishResult perform(Task task, TaskActionToolbox toolbox) return retVal; } + private void checkWithSegmentLock() + { + final Map> oldSegmentsMap = groupSegmentsByIntervalAndSort(segmentsToBeOverwritten); + final Map> newSegmentsMap = groupSegmentsByIntervalAndSort(segments); + + oldSegmentsMap.values().forEach(SegmentLockHelper::verifyRootPartitionIsAdjacentAndAtomicUpdateGroupIsFull); + newSegmentsMap.values().forEach(SegmentLockHelper::verifyRootPartitionIsAdjacentAndAtomicUpdateGroupIsFull); + + oldSegmentsMap.forEach((interval, oldSegmentsPerInterval) -> { + final List newSegmentsPerInterval = Preconditions.checkNotNull( + newSegmentsMap.get(interval), + "segments of interval[%s]", + interval + ); + // These lists are already sorted in groupSegmentsByIntervalAndSort(). + final int oldStartRootPartitionId = oldSegmentsPerInterval.get(0).getStartRootPartitionId(); + final int oldEndRootPartitionId = oldSegmentsPerInterval.get(oldSegmentsPerInterval.size() - 1) + .getEndRootPartitionId(); + final int newStartRootPartitionId = newSegmentsPerInterval.get(0).getStartRootPartitionId(); + final int newEndRootPartitionId = newSegmentsPerInterval.get(newSegmentsPerInterval.size() - 1) + .getEndRootPartitionId(); + + if (oldStartRootPartitionId != newStartRootPartitionId || oldEndRootPartitionId != newEndRootPartitionId) { + throw new ISE( + "Root partition range[%d, %d] of new segments doesn't match to root partition range[%d, %d] of old segments", + newStartRootPartitionId, + newEndRootPartitionId, + oldStartRootPartitionId, + oldEndRootPartitionId + ); + } + + newSegmentsPerInterval + .forEach(eachNewSegment -> oldSegmentsPerInterval + .forEach(eachOldSegment -> { + if (eachNewSegment.getMinorVersion() <= eachOldSegment.getMinorVersion()) { + throw new ISE( + "New segment[%s] have a smaller minor version than old segment[%s]", + eachNewSegment, + eachOldSegment + ); + } + })); + }); + } + + private static Map> groupSegmentsByIntervalAndSort(Set segments) + { + final Map> segmentsMap = new HashMap<>(); + segments.forEach(segment -> segmentsMap.computeIfAbsent(segment.getInterval(), k -> new ArrayList<>()) + .add(segment)); + segmentsMap.values().forEach(segmentsPerInterval -> segmentsPerInterval.sort((s1, s2) -> { + if (s1.getStartRootPartitionId() != s2.getStartRootPartitionId()) { + return Integer.compare(s1.getStartRootPartitionId(), s2.getStartRootPartitionId()); + } else { + return Integer.compare(s1.getEndRootPartitionId(), s2.getEndRootPartitionId()); + } + })); + return segmentsMap; + } + @Override public boolean isAudited() { @@ -159,7 +267,8 @@ public boolean isAudited() public String toString() { return "SegmentTransactionalInsertAction{" + - "segments=" + segments + + "segmentsToBeOverwritten=" + segmentsToBeOverwritten + + ", segments=" + segments + ", startMetadata=" + startMetadata + ", endMetadata=" + endMetadata + '}'; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAction.java index 206fd5fde7be..e268188a464e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAction.java @@ -26,8 +26,10 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { - @JsonSubTypes.Type(name = "lockAcquire", value = LockAcquireAction.class), - @JsonSubTypes.Type(name = "lockTryAcquire", value = LockTryAcquireAction.class), + @JsonSubTypes.Type(name = "lockAcquire", value = TimeChunkLockAcquireAction.class), + @JsonSubTypes.Type(name = "lockTryAcquire", value = TimeChunkLockTryAcquireAction.class), + @JsonSubTypes.Type(name = "segmentLockTryAcquire", value = SegmentLockTryAcquireAction.class), + @JsonSubTypes.Type(name = "segmentLockAcquire", value = SegmentLockAcquireAction.class), @JsonSubTypes.Type(name = "lockList", value = LockListAction.class), @JsonSubTypes.Type(name = "lockRelease", value = LockReleaseAction.class), @JsonSubTypes.Type(name = "segmentInsertion", value = SegmentInsertAction.class), diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskActionPreconditions.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskActionPreconditions.java deleted file mode 100644 index 6db8d8e2ab5b..000000000000 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskActionPreconditions.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.indexing.common.actions; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.druid.indexing.common.TaskLock; -import org.apache.druid.indexing.common.task.Task; -import org.apache.druid.indexing.overlord.TaskLockbox; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.timeline.DataSegment; -import org.joda.time.DateTime; - -import java.util.Collection; -import java.util.List; -import java.util.Map.Entry; -import java.util.NavigableMap; -import java.util.TreeMap; - -public class TaskActionPreconditions -{ - static void checkLockCoversSegments( - final Task task, - final TaskLockbox taskLockbox, - final Collection segments - ) - { - if (!isLockCoversSegments(task, taskLockbox, segments)) { - throw new ISE( - "Segments[%s] are not covered by locks[%s] for task[%s]", - segments, - taskLockbox.findLocksForTask(task), - task.getId() - ); - } - } - - @VisibleForTesting - static boolean isLockCoversSegments( - final Task task, - final TaskLockbox taskLockbox, - final Collection segments - ) - { - // Verify that each of these segments falls under some lock - - // NOTE: It is possible for our lock to be revoked (if the task has failed and given up its locks) after we check - // NOTE: it and before we perform the segment insert, but, that should be OK since the worst that happens is we - // NOTE: insert some segments from the task but not others. - - final NavigableMap taskLockMap = getTaskLockMap(taskLockbox, task); - if (taskLockMap.isEmpty()) { - return false; - } - - return segments.stream().allMatch( - segment -> { - final Entry entry = taskLockMap.floorEntry(segment.getInterval().getStart()); - if (entry == null) { - return false; - } - - final TaskLock taskLock = entry.getValue(); - return taskLock.getInterval().contains(segment.getInterval()) && - taskLock.getDataSource().equals(segment.getDataSource()) && - taskLock.getVersion().compareTo(segment.getVersion()) >= 0; - } - ); - } - - private static NavigableMap getTaskLockMap(TaskLockbox taskLockbox, Task task) - { - final List taskLocks = taskLockbox.findLocksForTask(task); - final NavigableMap taskLockMap = new TreeMap<>(); - taskLocks.forEach(taskLock -> taskLockMap.put(taskLock.getInterval().getStart(), taskLock)); - return taskLockMap; - } -} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskLocks.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskLocks.java new file mode 100644 index 000000000000..a7feb4fc5a2a --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskLocks.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.actions; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.SegmentLock; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TimeChunkLock; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.TaskLockbox; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.timeline.DataSegment; +import org.joda.time.DateTime; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.TreeMap; + +public class TaskLocks +{ + static void checkLockCoversSegments( + final Task task, + final TaskLockbox taskLockbox, + final Collection segments + ) + { + if (!isLockCoversSegments(task, taskLockbox, segments)) { + throw new ISE( + "Segments[%s] are not covered by locks[%s] for task[%s]", + segments, + taskLockbox.findLocksForTask(task), + task.getId() + ); + } + } + + @VisibleForTesting + static boolean isLockCoversSegments( + final Task task, + final TaskLockbox taskLockbox, + final Collection segments + ) + { + // Verify that each of these segments falls under some lock + + // NOTE: It is possible for our lock to be revoked (if the task has failed and given up its locks) after we check + // NOTE: it and before we perform the segment insert, but, that should be OK since the worst that happens is we + // NOTE: insert some segments from the task but not others. + + final NavigableMap> taskLockMap = getTaskLockMap(taskLockbox, task); + if (taskLockMap.isEmpty()) { + return false; + } + + return isLockCoversSegments(taskLockMap, segments); + } + + public static boolean isLockCoversSegments( + NavigableMap> taskLockMap, + Collection segments + ) + { + return segments.stream().allMatch( + segment -> { + final Entry> entry = taskLockMap.floorEntry(segment.getInterval().getStart()); + if (entry == null) { + return false; + } + + final List locks = entry.getValue(); + return locks.stream().anyMatch( + lock -> { + if (lock.getGranularity() == LockGranularity.TIME_CHUNK) { + final TimeChunkLock timeChunkLock = (TimeChunkLock) lock; + return timeChunkLock.getInterval().contains(segment.getInterval()) + && timeChunkLock.getDataSource().equals(segment.getDataSource()) + && timeChunkLock.getVersion().compareTo(segment.getVersion()) >= 0; + } else { + final SegmentLock segmentLock = (SegmentLock) lock; + return segmentLock.getInterval().contains(segment.getInterval()) + && segmentLock.getDataSource().equals(segment.getDataSource()) + && segmentLock.getVersion().compareTo(segment.getVersion()) >= 0 + && segmentLock.getPartitionId() == segment.getShardSpec().getPartitionNum(); + } + } + ); + } + ); + } + + public static List findLocksForSegments( + final Task task, + final TaskLockbox taskLockbox, + final Collection segments + ) + { + final NavigableMap> taskLockMap = getTaskLockMap(taskLockbox, task); + if (taskLockMap.isEmpty()) { + return Collections.emptyList(); + } + + final List found = new ArrayList<>(); + segments.forEach(segment -> { + final Entry> entry = taskLockMap.floorEntry(segment.getInterval().getStart()); + if (entry == null) { + throw new ISE("Can't find lock for the interval of segment[%s]", segment.getId()); + } + + final List locks = entry.getValue(); + locks.forEach(lock -> { + if (lock.getGranularity() == LockGranularity.TIME_CHUNK) { + final TimeChunkLock timeChunkLock = (TimeChunkLock) lock; + if (timeChunkLock.getInterval().contains(segment.getInterval()) + && timeChunkLock.getDataSource().equals(segment.getDataSource()) + && timeChunkLock.getVersion().compareTo(segment.getVersion()) >= 0) { + found.add(lock); + } + } else { + final SegmentLock segmentLock = (SegmentLock) lock; + if (segmentLock.getInterval().contains(segment.getInterval()) + && segmentLock.getDataSource().equals(segment.getDataSource()) + && segmentLock.getVersion().compareTo(segment.getVersion()) >= 0 + && segmentLock.getPartitionId() == segment.getShardSpec().getPartitionNum()) { + found.add(lock); + } + } + }); + }); + return found; + } + + private static NavigableMap> getTaskLockMap(TaskLockbox taskLockbox, Task task) + { + final List taskLocks = taskLockbox.findLocksForTask(task); + final NavigableMap> taskLockMap = new TreeMap<>(); + taskLocks.forEach(taskLock -> taskLockMap.computeIfAbsent(taskLock.getInterval().getStart(), k -> new ArrayList<>()) + .add(taskLock)); + return taskLockMap; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LockAcquireAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TimeChunkLockAcquireAction.java similarity index 75% rename from indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LockAcquireAction.java rename to indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TimeChunkLockAcquireAction.java index 0b2e49d6e501..83eec1351522 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LockAcquireAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TimeChunkLockAcquireAction.java @@ -28,11 +28,17 @@ import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.LockResult; +import org.apache.druid.indexing.overlord.TimeChunkLockRequest; import org.joda.time.Interval; import javax.annotation.Nullable; -public class LockAcquireAction implements TaskAction +/** + * TaskAction to acquire a {@link org.apache.druid.indexing.common.TimeChunkLock}. + * This action is a blocking operation and the caller could wait until it gets {@link TaskLock} + * (up to timeoutMs if it's > 0). It returns null if it fails to get a lock within timeout. + */ +public class TimeChunkLockAcquireAction implements TaskAction { private final TaskLockType type; @@ -43,7 +49,7 @@ public class LockAcquireAction implements TaskAction private final long timeoutMs; @JsonCreator - public LockAcquireAction( + public TimeChunkLockAcquireAction( @JsonProperty("lockType") @Nullable TaskLockType type, // nullable for backward compatibility @JsonProperty("interval") Interval interval, @JsonProperty("timeoutMs") long timeoutMs @@ -84,9 +90,11 @@ public TypeReference getReturnTypeReference() public TaskLock perform(Task task, TaskActionToolbox toolbox) { try { - final LockResult result = timeoutMs == 0 ? - toolbox.getTaskLockbox().lock(type, task, interval) : - toolbox.getTaskLockbox().lock(type, task, interval, timeoutMs); + final LockResult result = timeoutMs == 0 + ? toolbox.getTaskLockbox() + .lock(task, new TimeChunkLockRequest(type, task, interval, null)) + : toolbox.getTaskLockbox() + .lock(task, new TimeChunkLockRequest(type, task, interval, null), timeoutMs); return result.isOk() ? result.getTaskLock() : null; } catch (InterruptedException e) { @@ -103,7 +111,7 @@ public boolean isAudited() @Override public String toString() { - return "LockAcquireAction{" + + return "TimeChunkLockAcquireAction{" + "lockType=" + type + ", interval=" + interval + ", timeoutMs=" + timeoutMs + diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LockTryAcquireAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TimeChunkLockTryAcquireAction.java similarity index 79% rename from indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LockTryAcquireAction.java rename to indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TimeChunkLockTryAcquireAction.java index f24266ba4b9a..7c2b7e0bf98c 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LockTryAcquireAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TimeChunkLockTryAcquireAction.java @@ -27,11 +27,16 @@ import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.LockResult; +import org.apache.druid.indexing.overlord.TimeChunkLockRequest; import org.joda.time.Interval; import javax.annotation.Nullable; -public class LockTryAcquireAction implements TaskAction +/** + * TaskAction to try to acquire a {@link org.apache.druid.indexing.common.TimeChunkLock}. + * This action returns null immediately if it fails to get a lock for the given interval. + */ +public class TimeChunkLockTryAcquireAction implements TaskAction { @JsonIgnore private final TaskLockType type; @@ -40,7 +45,7 @@ public class LockTryAcquireAction implements TaskAction private final Interval interval; @JsonCreator - public LockTryAcquireAction( + public TimeChunkLockTryAcquireAction( @JsonProperty("lockType") @Nullable TaskLockType type, // nullable for backward compatibility @JsonProperty("interval") Interval interval ) @@ -72,7 +77,10 @@ public TypeReference getReturnTypeReference() @Override public TaskLock perform(Task task, TaskActionToolbox toolbox) { - final LockResult result = toolbox.getTaskLockbox().tryLock(type, task, interval); + final LockResult result = toolbox.getTaskLockbox().tryLock( + task, + new TimeChunkLockRequest(type, task, interval, null) + ); return result.isOk() ? result.getTaskLock() : null; } @@ -85,8 +93,8 @@ public boolean isAudited() @Override public String toString() { - return "LockTryAcquireAction{" + - "lockType=" + type + + return "TimeChunkLockTryAcquireAction{" + + ", type=" + type + ", interval=" + interval + '}'; } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/index/YeOldePlumberSchool.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/index/YeOldePlumberSchool.java index f872042049ae..7d3e9de651b5 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/index/YeOldePlumberSchool.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/index/YeOldePlumberSchool.java @@ -108,7 +108,8 @@ public Plumber findPlumber( config.getMaxRowsInMemory(), TuningConfigs.getMaxBytesInMemoryOrDefault(config.getMaxBytesInMemory()), config.isReportParseExceptions(), - config.getDedupColumn() + config.getDedupColumn(), + null ); // Temporary directory to hold spilled segments. diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java new file mode 100644 index 000000000000..b00c646385a3 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.task; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import org.apache.druid.data.input.FirehoseFactory; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.actions.SegmentListUsedAction; +import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction; +import org.apache.druid.indexing.firehose.IngestSegmentFirehoseFactory; +import org.apache.druid.indexing.firehose.WindowedSegmentId; +import org.apache.druid.java.util.common.JodaUtils; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.java.util.common.granularity.GranularityType; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.indexing.granularity.GranularitySpec; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.TimelineObjectHolder; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.PartitionChunk; +import org.joda.time.Interval; +import org.joda.time.Period; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Abstract class for batch tasks like {@link IndexTask}. + * Provides some methods such as {@link #determineSegmentGranularity}, {@link #findInputSegments}, + * and {@link #determineLockGranularityandTryLock} for easily acquiring task locks. + */ +public abstract class AbstractBatchIndexTask extends AbstractTask +{ + private static final Logger log = new Logger(AbstractBatchIndexTask.class); + + private final SegmentLockHelper segmentLockHelper; + + /** + * State to indicate that this task will use segmentLock or timeChunkLock. + * This is automatically set when {@link #determineLockGranularityandTryLock} is called. + */ + private boolean useSegmentLock; + + protected AbstractBatchIndexTask(String id, String dataSource, Map context) + { + super(id, dataSource, context); + segmentLockHelper = new SegmentLockHelper(); + } + + protected AbstractBatchIndexTask( + String id, + @Nullable String groupId, + @Nullable TaskResource taskResource, + String dataSource, + @Nullable Map context + ) + { + super(id, groupId, taskResource, dataSource, context); + segmentLockHelper = new SegmentLockHelper(); + } + + /** + * Return true if this task can overwrite existing segments. + */ + public abstract boolean requireLockExistingSegments(); + + /** + * Find segments to lock in the given intervals. + * If this task is intend to overwrite only some segments in those intervals, this method should return only those + * segments instead of entire segments in those intervals. + */ + public abstract List findSegmentsToLock(TaskActionClient taskActionClient, List intervals) + throws IOException; + + /** + * Returns true if this task is in the perfect (guaranteed) rollup mode. + */ + public abstract boolean isPerfectRollup(); + + /** + * Returns the segmentGranularity defined in the ingestion spec. + */ + @Nullable + public abstract Granularity getSegmentGranularity(); + + public boolean isUseSegmentLock() + { + return useSegmentLock; + } + + public SegmentLockHelper getSegmentLockHelper() + { + return segmentLockHelper; + } + + /** + * Determine lockGranularity to use and try to acquire necessary locks. + * This method respects the value of 'forceTimeChunkLock' in task context. + * If it's set to false or missing, this method checks if this task can use segmentLock. + */ + protected boolean determineLockGranularityAndTryLock( + TaskActionClient client, + GranularitySpec granularitySpec + ) throws IOException + { + final List intervals = granularitySpec.bucketIntervals().isPresent() + ? new ArrayList<>(granularitySpec.bucketIntervals().get()) + : Collections.emptyList(); + return determineLockGranularityandTryLock(client, intervals); + } + + boolean determineLockGranularityandTryLock(TaskActionClient client, List intervals) throws IOException + { + final boolean forceTimeChunkLock = getContextValue( + Tasks.FORCE_TIME_CHUNK_LOCK_KEY, + Tasks.DEFAULT_FORCE_TIME_CHUNK_LOCK + ); + // Respect task context value most. + if (forceTimeChunkLock) { + log.info("[%s] is set to true in task context. Use timeChunk lock", Tasks.FORCE_TIME_CHUNK_LOCK_KEY); + useSegmentLock = false; + if (!intervals.isEmpty()) { + return tryTimeChunkLock(client, intervals); + } else { + return true; + } + } else { + if (!intervals.isEmpty()) { + final LockGranularityDetermineResult result = determineSegmentGranularity(client, intervals); + useSegmentLock = result.lockGranularity == LockGranularity.SEGMENT; + return tryLockWithDetermineResult(client, result); + } else { + return true; + } + } + } + + boolean determineLockGranularityandTryLockWithSegments(TaskActionClient client, List segments) + throws IOException + { + final boolean forceTimeChunkLock = getContextValue( + Tasks.FORCE_TIME_CHUNK_LOCK_KEY, + Tasks.DEFAULT_FORCE_TIME_CHUNK_LOCK + ); + if (forceTimeChunkLock) { + log.info("[%s] is set to true in task context. Use timeChunk lock", Tasks.FORCE_TIME_CHUNK_LOCK_KEY); + useSegmentLock = false; + return tryTimeChunkLock( + client, + new ArrayList<>(segments.stream().map(DataSegment::getInterval).collect(Collectors.toSet())) + ); + } else { + final LockGranularityDetermineResult result = determineSegmentGranularity(segments); + useSegmentLock = result.lockGranularity == LockGranularity.SEGMENT; + return tryLockWithDetermineResult(client, result); + } + } + + private LockGranularityDetermineResult determineSegmentGranularity(TaskActionClient client, List intervals) + throws IOException + { + if (requireLockExistingSegments()) { + if (isPerfectRollup()) { + log.info("Using timeChunk lock for perfect rollup"); + return new LockGranularityDetermineResult(LockGranularity.TIME_CHUNK, intervals, null); + } else if (!intervals.isEmpty()) { + // This method finds segments falling in all given intervals and then tries to lock those segments. + // Thus, there might be a race between calling findSegmentsToLock() and determineSegmentGranularity(), + // i.e., a new segment can be added to the interval or an existing segment might be removed. + // Removed segments should be fine because indexing tasks would do nothing with removed segments. + // However, tasks wouldn't know about new segments added after findSegmentsToLock() call, it may missing those + // segments. This is usually fine, but if you want to avoid this, you should use timeChunk lock instead. + return determineSegmentGranularity(findSegmentsToLock(client, intervals)); + } else { + log.info("Using segment lock for empty intervals"); + return new LockGranularityDetermineResult(LockGranularity.SEGMENT, null, Collections.emptyList()); + } + } else { + log.info("Using segment lock since we don't have to lock existing segments"); + return new LockGranularityDetermineResult(LockGranularity.SEGMENT, null, Collections.emptyList()); + } + } + + private boolean tryLockWithDetermineResult(TaskActionClient client, LockGranularityDetermineResult result) + throws IOException + { + if (result.lockGranularity == LockGranularity.TIME_CHUNK) { + return tryTimeChunkLock(client, Preconditions.checkNotNull(result.intervals, "intervals")); + } else { + return segmentLockHelper.verifyAndLockExistingSegments( + client, + Preconditions.checkNotNull(result.segments, "segments") + ); + } + } + + private boolean tryTimeChunkLock(TaskActionClient client, List intervals) throws IOException + { + // In this case, the intervals to lock must be aligned with segmentGranularity if it's defined + final Set uniqueIntervals = new HashSet<>(); + for (Interval interval : JodaUtils.condenseIntervals(intervals)) { + final Granularity segmentGranularity = getSegmentGranularity(); + if (segmentGranularity == null) { + uniqueIntervals.add(interval); + } else { + Iterables.addAll(uniqueIntervals, segmentGranularity.getIterable(interval)); + } + } + + for (Interval interval : uniqueIntervals) { + final TaskLock lock = client.submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)); + if (lock == null) { + return false; + } + } + return true; + } + + private LockGranularityDetermineResult determineSegmentGranularity(List segments) + { + if (segments.isEmpty()) { + log.info("Using segment lock for empty segments"); + // Set useSegmentLock even though we don't get any locks. + // Note that we should get any lock before data ingestion if we are supposed to use timChunk lock. + return new LockGranularityDetermineResult(LockGranularity.SEGMENT, null, Collections.emptyList()); + } + + if (requireLockExistingSegments()) { + final Granularity granularityFromSegments = findGranularityFromSegments(segments); + @Nullable + final Granularity segmentGranularityFromSpec = getSegmentGranularity(); + final List intervals = segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()); + + if (granularityFromSegments == null + || segmentGranularityFromSpec != null + && (!granularityFromSegments.equals(segmentGranularityFromSpec) + || segments.stream().anyMatch(segment -> !segmentGranularityFromSpec.isAligned(segment.getInterval())))) { + // This case is one of the followings: + // 1) Segments have different granularities. + // 2) Segment granularity in ingestion spec is different from the one of existig segments. + // 3) Some existing segments are not aligned with the segment granularity in the ingestion spec. + log.info("Detected segmentGranularity change. Using timeChunk lock"); + return new LockGranularityDetermineResult(LockGranularity.TIME_CHUNK, intervals, null); + } else { + // Use segment lock + // Create a timeline to find latest segments only + final VersionedIntervalTimeline timeline = VersionedIntervalTimeline.forSegments( + segments + ); + + final List segmentsToLock = timeline + .lookup(JodaUtils.umbrellaInterval(intervals)) + .stream() + .map(TimelineObjectHolder::getObject) + .flatMap(partitionHolder -> StreamSupport.stream(partitionHolder.spliterator(), false)) + .map(PartitionChunk::getObject) + .collect(Collectors.toList()); + log.info("No segmentGranularity change detected and it's not perfect rollup. Using segment lock"); + return new LockGranularityDetermineResult(LockGranularity.SEGMENT, null, segmentsToLock); + } + } else { + // Set useSegmentLock even though we don't get any locks. + // Note that we should get any lock before data ingestion if we are supposed to use timChunk lock. + log.info("Using segment lock since we don't have to lock existing segments"); + return new LockGranularityDetermineResult(LockGranularity.SEGMENT, null, Collections.emptyList()); + } + } + + @Nullable + static Granularity findGranularityFromSegments(List segments) + { + if (segments.isEmpty()) { + return null; + } + final Period firstSegmentPeriod = segments.get(0).getInterval().toPeriod(); + final boolean allHasSameGranularity = segments + .stream() + .allMatch(segment -> firstSegmentPeriod.equals(segment.getInterval().toPeriod())); + if (allHasSameGranularity) { + return GranularityType.fromPeriod(firstSegmentPeriod).getDefaultGranularity(); + } else { + return null; + } + } + + /** + * If the given firehoseFactory is {@link IngestSegmentFirehoseFactory}, then it finds the segments to lock + * from the firehoseFactory. This is because those segments will be read by this task no matter what segments would be + * filtered by intervalsToRead, so they need to be locked. + * + * However, firehoseFactory is not IngestSegmentFirehoseFactory, it means this task will overwrite some segments + * with data read from some input source outside of Druid. As a result, only the segments falling in intervalsToRead + * should be locked. + */ + protected static List findInputSegments( + String dataSource, + TaskActionClient actionClient, + List intervalsToRead, + FirehoseFactory firehoseFactory + ) throws IOException + { + if (firehoseFactory instanceof IngestSegmentFirehoseFactory) { + // intervalsToRead is ignored here. + final List inputSegments = ((IngestSegmentFirehoseFactory) firehoseFactory).getSegments(); + if (inputSegments == null) { + final Interval inputInterval = Preconditions.checkNotNull( + ((IngestSegmentFirehoseFactory) firehoseFactory).getInterval(), + "input interval" + ); + + return actionClient.submit( + new SegmentListUsedAction(dataSource, null, Collections.singletonList(inputInterval)) + ); + } else { + final List inputSegmentIds = inputSegments.stream() + .map(WindowedSegmentId::getSegmentId) + .collect(Collectors.toList()); + final List dataSegmentsInIntervals = actionClient.submit( + new SegmentListUsedAction( + dataSource, + null, + inputSegments.stream() + .flatMap(windowedSegmentId -> windowedSegmentId.getIntervals().stream()) + .collect(Collectors.toSet()) + ) + ); + return dataSegmentsInIntervals.stream() + .filter(segment -> inputSegmentIds.contains(segment.getId().toString())) + .collect(Collectors.toList()); + } + } else { + return actionClient.submit(new SegmentListUsedAction(dataSource, null, intervalsToRead)); + } + } + + private static class LockGranularityDetermineResult + { + private final LockGranularity lockGranularity; + @Nullable + private final List intervals; // null for segmentLock + @Nullable + private final List segments; // null for timeChunkLock + + private LockGranularityDetermineResult( + LockGranularity lockGranularity, + @Nullable List intervals, + @Nullable List segments + ) + { + this.lockGranularity = lockGranularity; + this.intervals = intervals; + this.segments = segments; + } + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractFixedIntervalTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractFixedIntervalTask.java index 6959677e7812..9582b987876c 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractFixedIntervalTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractFixedIntervalTask.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import org.apache.druid.indexing.common.TaskLockType; -import org.apache.druid.indexing.common.actions.LockTryAcquireAction; import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction; import org.joda.time.Interval; import java.util.Map; @@ -72,7 +72,7 @@ protected AbstractFixedIntervalTask( @Override public boolean isReady(TaskActionClient taskActionClient) throws Exception { - return taskActionClient.submit(new LockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)) != null; + return taskActionClient.submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)) != null; } @JsonProperty diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java index 27e585e326fd..370b30ed1ec8 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java @@ -188,7 +188,7 @@ static String joinId(List objects) return ID_JOINER.join(objects); } - static String joinId(Object...objects) + static String joinId(Object... objects) { return ID_JOINER.join(objects); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTask.java index 81f4797d08f0..2d5867b8981b 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTask.java @@ -44,12 +44,16 @@ import org.apache.druid.indexing.appenderator.ActionBasedUsedSegmentChecker; import org.apache.druid.indexing.common.IngestionStatsAndErrorsTaskReport; import org.apache.druid.indexing.common.IngestionStatsAndErrorsTaskReportData; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskRealtimeMetricsMonitorBuilder; import org.apache.druid.indexing.common.TaskReport; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.actions.SegmentAllocateAction; +import org.apache.druid.indexing.common.actions.SegmentLockAcquireAction; import org.apache.druid.indexing.common.actions.SegmentTransactionalInsertAction; import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TimeChunkLockAcquireAction; import org.apache.druid.indexing.common.config.TaskConfig; import org.apache.druid.indexing.common.index.RealtimeAppenderatorIngestionSpec; import org.apache.druid.indexing.common.index.RealtimeAppenderatorTuningConfig; @@ -84,6 +88,7 @@ import org.apache.druid.segment.realtime.plumber.Committers; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthorizerMapper; +import org.apache.druid.timeline.partition.NumberedShardSpecFactory; import org.apache.druid.utils.CircularBuffer; import javax.servlet.http.HttpServletRequest; @@ -94,6 +99,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.File; +import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -160,6 +166,9 @@ private static String makeTaskId(RealtimeAppenderatorIngestionSpec spec) @JsonIgnore private final AuthorizerMapper authorizerMapper; + @JsonIgnore + private final LockGranularity lockGranularity; + @JsonIgnore private IngestionState ingestionState; @@ -195,6 +204,9 @@ public AppenderatorDriverRealtimeIndexTask( this.ingestionState = IngestionState.NOT_STARTED; this.rowIngestionMeters = rowIngestionMetersFactory.createRowIngestionMeters(); + this.lockGranularity = getContextValue(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, Tasks.DEFAULT_FORCE_TIME_CHUNK_LOCK) + ? LockGranularity.TIME_CHUNK + : LockGranularity.SEGMENT; } @Override @@ -274,7 +286,34 @@ public TaskStatus run(final TaskToolbox toolbox) toolbox.getDataSegmentServerAnnouncer().announce(); toolbox.getDruidNodeAnnouncer().announce(discoveryDruidNode); - driver.startJob(); + driver.startJob( + segmentId -> { + try { + if (lockGranularity == LockGranularity.SEGMENT) { + return toolbox.getTaskActionClient().submit( + new SegmentLockAcquireAction( + TaskLockType.EXCLUSIVE, + segmentId.getInterval(), + segmentId.getVersion(), + segmentId.getShardSpec().getPartitionNum(), + 1000L + ) + ).isOk(); + } else { + return toolbox.getTaskActionClient().submit( + new TimeChunkLockAcquireAction( + TaskLockType.EXCLUSIVE, + segmentId.getInterval(), + 1000L + ) + ) != null; + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + ); // Set up metrics emission toolbox.getMonitorScheduler().addMonitor(metricsMonitor); @@ -289,8 +328,15 @@ public TaskStatus run(final TaskToolbox toolbox) int sequenceNumber = 0; String sequenceName = makeSequenceName(getId(), sequenceNumber); - final TransactionalSegmentPublisher publisher = (segments, commitMetadata) -> { - final SegmentTransactionalInsertAction action = new SegmentTransactionalInsertAction(segments); + final TransactionalSegmentPublisher publisher = (mustBeNullOrEmptySegments, segments, commitMetadata) -> { + if (mustBeNullOrEmptySegments != null && !mustBeNullOrEmptySegments.isEmpty()) { + throw new ISE("WTH? stream ingestion tasks are overwriting segments[%s]", mustBeNullOrEmptySegments); + } + final SegmentTransactionalInsertAction action = SegmentTransactionalInsertAction.appendAction( + segments, + null, + null + ); return toolbox.getTaskActionClient().submit(action); }; @@ -726,7 +772,9 @@ private static StreamAppenderatorDriver newDriver( schema.getGranularitySpec().getSegmentGranularity(), sequenceName, previousSegmentId, - skipSegmentLineageCheck + skipSegmentLineageCheck, + NumberedShardSpecFactory.instance(), + LockGranularity.TIME_CHUNK ) ), toolbox.getSegmentHandoffNotifierFactory(), diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CachingLocalSegmentAllocator.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CachingLocalSegmentAllocator.java new file mode 100644 index 000000000000..8d5680e6ed11 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CachingLocalSegmentAllocator.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.task; + +import com.google.common.base.Preconditions; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskToolbox; +import org.apache.druid.indexing.common.actions.LockListAction; +import org.apache.druid.indexing.common.task.IndexTask.ShardSpecs; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; +import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.timeline.partition.ShardSpecFactory; +import org.joda.time.Interval; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Allocates all necessary segments locally at the beginning and reuse them. + */ +class CachingLocalSegmentAllocator implements IndexTaskSegmentAllocator +{ + private final TaskToolbox toolbox; + private final String taskId; + private final String dataSource; + private final Map> allocateSpec; + @Nullable + private final ShardSpecs shardSpecs; + + // sequenceName -> segmentId + private final Map sequenceNameToSegmentId; + + CachingLocalSegmentAllocator( + TaskToolbox toolbox, + String taskId, + String dataSource, + Map> allocateSpec + ) throws IOException + { + this.toolbox = toolbox; + this.taskId = taskId; + this.dataSource = dataSource; + this.allocateSpec = allocateSpec; + this.sequenceNameToSegmentId = new HashMap<>(); + + final Map> intervalToIds = getIntervalToSegmentIds(); + final Map> shardSpecMap = new HashMap<>(); + + for (Map.Entry> entry : intervalToIds.entrySet()) { + final Interval interval = entry.getKey(); + final List idsPerInterval = intervalToIds.get(interval); + + for (SegmentIdWithShardSpec segmentIdentifier : idsPerInterval) { + shardSpecMap.computeIfAbsent(interval, k -> new ArrayList<>()).add(segmentIdentifier.getShardSpec()); + // The shardSpecs for partitioning and publishing can be different if isExtendableShardSpecs = true. + sequenceNameToSegmentId.put(getSequenceName(interval, segmentIdentifier.getShardSpec()), segmentIdentifier); + } + } + shardSpecs = new ShardSpecs(shardSpecMap); + } + + private Map> getIntervalToSegmentIds() throws IOException + { + final Map intervalToVersion = getToolbox().getTaskActionClient() + .submit(new LockListAction()) + .stream() + .collect(Collectors.toMap(TaskLock::getInterval, TaskLock::getVersion)); + final Map> allocateSpec = getAllocateSpec(); + final Map> intervalToSegmentIds = new HashMap<>(allocateSpec.size()); + for (Entry> entry : allocateSpec.entrySet()) { + final Interval interval = entry.getKey(); + final ShardSpecFactory shardSpecFactory = entry.getValue().lhs; + final int numSegmentsToAllocate = Preconditions.checkNotNull( + entry.getValue().rhs, + "numSegmentsToAllocate for interval[%s]", + interval + ); + + intervalToSegmentIds.put( + interval, + IntStream.range(0, numSegmentsToAllocate) + .mapToObj(i -> new SegmentIdWithShardSpec( + getDataSource(), + interval, + findVersion(intervalToVersion, interval), + shardSpecFactory.create(getToolbox().getObjectMapper(), i) + )) + .collect(Collectors.toList()) + ); + } + return intervalToSegmentIds; + } + + private static String findVersion(Map intervalToVersion, Interval interval) + { + return intervalToVersion.entrySet().stream() + .filter(entry -> entry.getKey().contains(interval)) + .map(Entry::getValue) + .findFirst() + .orElseThrow(() -> new ISE("Cannot find a version for interval[%s]", interval)); + } + + + TaskToolbox getToolbox() + { + return toolbox; + } + + String getTaskId() + { + return taskId; + } + + String getDataSource() + { + return dataSource; + } + + Map> getAllocateSpec() + { + return allocateSpec; + } + + @Override + public SegmentIdWithShardSpec allocate( + InputRow row, + String sequenceName, + String previousSegmentId, + boolean skipSegmentLineageCheck + ) + { + return sequenceNameToSegmentId.get(sequenceName); + } + + @Override + public String getSequenceName(Interval interval, InputRow inputRow) + { + // Sequence name is based solely on the shardSpec, and there will only be one segment per sequence. + return getSequenceName(interval, shardSpecs.getShardSpec(interval, inputRow)); + } + + /** + * Create a sequence name from the given shardSpec and interval. + * + * See {@link org.apache.druid.timeline.partition.HashBasedNumberedShardSpec} as an example of partitioning. + */ + private String getSequenceName(Interval interval, ShardSpec shardSpec) + { + return StringUtils.format("%s_%s_%d", taskId, interval, shardSpec.getPartitionNum()); + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java index 6a0ae54d8494..5d3bdf842b55 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java @@ -96,14 +96,12 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.SortedSet; import java.util.TreeMap; -import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.StreamSupport; -public class CompactionTask extends AbstractTask +public class CompactionTask extends AbstractBatchIndexTask { private static final Logger log = new Logger(CompactionTask.class); private static final String TYPE = "compact"; @@ -224,6 +222,7 @@ public AggregatorFactory[] getMetricsSpec() @JsonProperty @Nullable + @Override public Granularity getSegmentGranularity() { return segmentGranularity; @@ -255,25 +254,37 @@ public int getPriority() return getContextValue(Tasks.PRIORITY_KEY, Tasks.DEFAULT_MERGE_TASK_PRIORITY); } - @VisibleForTesting - SegmentProvider getSegmentProvider() + @Override + public boolean isReady(TaskActionClient taskActionClient) throws Exception { - return segmentProvider; + final List segments = segmentProvider.checkAndGetSegments(taskActionClient); + return determineLockGranularityandTryLockWithSegments(taskActionClient, segments); } @Override - public boolean isReady(TaskActionClient taskActionClient) throws Exception + public boolean requireLockExistingSegments() + { + return true; + } + + @Override + public List findSegmentsToLock(TaskActionClient taskActionClient, List intervals) + throws IOException + { + return taskActionClient.submit(new SegmentListUsedAction(getDataSource(), null, intervals)); + } + + @Override + public boolean isPerfectRollup() { - final SortedSet intervals = new TreeSet<>(Comparators.intervalsByStartThenEnd()); - intervals.add(segmentProvider.interval); - return IndexTask.isReady(taskActionClient, intervals); + return tuningConfig != null && tuningConfig.isForceGuaranteedRollup(); } @Override public TaskStatus run(final TaskToolbox toolbox) throws Exception { if (indexTaskSpecs == null) { - indexTaskSpecs = createIngestionSchema( + final List ingestionSpecs = createIngestionSchema( toolbox, segmentProvider, partitionConfigurationManager, @@ -284,19 +295,21 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception coordinatorClient, segmentLoaderFactory, retryPolicyFactory - ).stream() - .map(spec -> new IndexTask( - getId(), - getGroupId(), - getTaskResource(), - getDataSource(), - spec, - getContext(), - authorizerMapper, - chatHandlerProvider, - rowIngestionMetersFactory - )) - .collect(Collectors.toList()); + ); + indexTaskSpecs = IntStream + .range(0, ingestionSpecs.size()) + .mapToObj(i -> new IndexTask( + createIndexTaskSpecId(i), + getGroupId(), + getTaskResource(), + getDataSource(), + ingestionSpecs.get(i), + getContext(), + authorizerMapper, + chatHandlerProvider, + rowIngestionMetersFactory + )) + .collect(Collectors.toList()); } if (indexTaskSpecs.isEmpty()) { @@ -312,10 +325,15 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception log.info("Running indexSpec: " + json); try { - final TaskStatus eachResult = eachSpec.run(toolbox); - if (!eachResult.isSuccess()) { + if (eachSpec.isReady(toolbox.getTaskActionClient())) { + final TaskStatus eachResult = eachSpec.run(toolbox); + if (!eachResult.isSuccess()) { + failCnt++; + log.warn("Failed to run indexSpec: [%s].\nTrying the next indexSpec.", json); + } + } else { failCnt++; - log.warn("Failed to run indexSpec: [%s].\nTrying the next indexSpec.", json); + log.warn("indexSpec is not ready: [%s].\nTrying the next indexSpec.", json); } } catch (Exception e) { @@ -329,6 +347,11 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception } } + private String createIndexTaskSpecId(int i) + { + return StringUtils.format("%s_%d", getId(), i); + } + /** * Generate {@link IndexIngestionSpec} from input segments. * @@ -360,6 +383,7 @@ static List createIngestionSchema( } // find metadata for interval + // queryableIndexAndSegments is sorted by the interval of the dataSegment final List> queryableIndexAndSegments = loadSegments( timelineSegments, segmentFileMap, @@ -387,7 +411,6 @@ static List createIngestionSchema( final List> segmentsToCompact = entry.getValue(); final DataSchema dataSchema = createDataSchema( segmentProvider.dataSource, - interval, segmentsToCompact, dimensionsSpec, metricsSpec, @@ -416,7 +439,6 @@ static List createIngestionSchema( // given segment granularity final DataSchema dataSchema = createDataSchema( segmentProvider.dataSource, - segmentProvider.interval, queryableIndexAndSegments, dimensionsSpec, metricsSpec, @@ -474,7 +496,7 @@ private static Pair, List usedSegments = segmentProvider.checkAndGetSegments(toolbox); + final List usedSegments = segmentProvider.checkAndGetSegments(toolbox.getTaskActionClient()); final Map segmentFileMap = toolbox.fetchSegments(usedSegments); final List> timelineSegments = VersionedIntervalTimeline .forSegments(usedSegments) @@ -484,7 +506,6 @@ private static Pair, List> queryableIndexAndSegments, @Nullable DimensionsSpec dimensionsSpec, @Nullable AggregatorFactory[] metricsSpec, @@ -508,6 +529,10 @@ private static DataSchema createDataSchema( return isRollup != null && isRollup; }); + final Interval totalInterval = JodaUtils.umbrellaInterval( + queryableIndexAndSegments.stream().map(p -> p.rhs.getInterval()).collect(Collectors.toList()) + ); + final GranularitySpec granularitySpec = new UniformGranularitySpec( Preconditions.checkNotNull(segmentGranularity), Granularities.NONE, @@ -693,6 +718,7 @@ static class SegmentProvider { private final String dataSource; private final Interval interval; + @Nullable private final List segments; SegmentProvider(String dataSource, Interval interval) @@ -710,21 +736,22 @@ static class SegmentProvider segments.stream().allMatch(segment -> segment.getDataSource().equals(dataSource)), "segments should have the same dataSource" ); - this.segments = segments; this.dataSource = dataSource; + this.segments = segments; this.interval = JodaUtils.umbrellaInterval( segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()) ); } + @Nullable List getSegments() { return segments; } - List checkAndGetSegments(TaskToolbox toolbox) throws IOException + List checkAndGetSegments(TaskActionClient actionClient) throws IOException { - final List usedSegments = toolbox.getTaskActionClient().submit( + final List usedSegments = actionClient.submit( new SegmentListUsedAction(dataSource, interval, null) ); final TimelineLookup timeline = VersionedIntervalTimeline.forSegments(usedSegments); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java index 2571c576d2aa..f8728c0c0e77 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java @@ -45,16 +45,19 @@ import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskReport; import org.apache.druid.indexing.common.TaskToolbox; -import org.apache.druid.indexing.common.actions.LockAcquireAction; -import org.apache.druid.indexing.common.actions.LockTryAcquireAction; import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TimeChunkLockAcquireAction; +import org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction; import org.apache.druid.indexing.common.config.TaskConfig; import org.apache.druid.indexing.common.stats.RowIngestionMeters; import org.apache.druid.indexing.hadoop.OverlordActionBasedUsedSegmentLister; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; +import org.apache.druid.segment.indexing.granularity.GranularitySpec; import org.apache.druid.segment.realtime.firehose.ChatHandler; import org.apache.druid.segment.realtime.firehose.ChatHandlerProvider; import org.apache.druid.server.security.Action; @@ -64,6 +67,7 @@ import org.apache.hadoop.util.ToolRunner; import org.joda.time.Interval; +import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -197,12 +201,42 @@ public boolean isReady(TaskActionClient taskActionClient) throws Exception intervals.get() ) ); - return taskActionClient.submit(new LockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)) != null; + return taskActionClient.submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)) != null; } else { return true; } } + @Override + public boolean requireLockExistingSegments() + { + throw new UnsupportedOperationException(); + } + + @Override + public List findSegmentsToLock(TaskActionClient taskActionClient, List intervals) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isPerfectRollup() + { + return true; + } + + @Nullable + @Override + public Granularity getSegmentGranularity() + { + final GranularitySpec granularitySpec = spec.getDataSchema().getGranularitySpec(); + if (granularitySpec instanceof ArbitraryGranularitySpec) { + return null; + } else { + return granularitySpec.getSegmentGranularity(); + } + } + @JsonProperty("spec") public HadoopIngestionSpec getSpec() { @@ -344,7 +378,7 @@ private TaskStatus runInternal(TaskToolbox toolbox) throws Exception // Note: if lockTimeoutMs is larger than ServerConfig.maxIdleTime, the below line can incur http timeout error. final TaskLock lock = Preconditions.checkNotNull( toolbox.getTaskActionClient().submit( - new LockAcquireAction(TaskLockType.EXCLUSIVE, interval, lockTimeoutMs) + new TimeChunkLockAcquireAction(TaskLockType.EXCLUSIVE, interval, lockTimeoutMs) ), "Cannot acquire a lock for interval[%s]", interval ); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopTask.java index 768735d32e2c..f32ab28b38be 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopTask.java @@ -44,7 +44,7 @@ import java.util.Map; -public abstract class HadoopTask extends AbstractTask +public abstract class HadoopTask extends AbstractBatchIndexTask { private static final Logger log = new Logger(HadoopTask.class); private static final ExtensionsConfig extensionsConfig; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java index 07a507af140d..b6a4126f8fac 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java @@ -28,7 +28,6 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; @@ -40,15 +39,13 @@ import org.apache.druid.hll.HyperLogLogCollector; import org.apache.druid.indexer.IngestionState; import org.apache.druid.indexer.TaskStatus; -import org.apache.druid.indexing.appenderator.ActionBasedSegmentAllocator; import org.apache.druid.indexing.appenderator.ActionBasedUsedSegmentChecker; import org.apache.druid.indexing.common.IngestionStatsAndErrorsTaskReport; import org.apache.druid.indexing.common.IngestionStatsAndErrorsTaskReportData; -import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.TaskRealtimeMetricsMonitorBuilder; import org.apache.druid.indexing.common.TaskReport; import org.apache.druid.indexing.common.TaskToolbox; -import org.apache.druid.indexing.common.actions.SegmentAllocateAction; import org.apache.druid.indexing.common.actions.SegmentTransactionalInsertAction; import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.stats.RowIngestionMeters; @@ -57,6 +54,7 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.JodaUtils; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.guava.Comparators; @@ -68,6 +66,7 @@ import org.apache.druid.segment.indexing.IngestionSpec; import org.apache.druid.segment.indexing.RealtimeIOConfig; import org.apache.druid.segment.indexing.TuningConfig; +import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import org.apache.druid.segment.indexing.granularity.GranularitySpec; import org.apache.druid.segment.realtime.FireDepartment; import org.apache.druid.segment.realtime.FireDepartmentMetrics; @@ -78,7 +77,6 @@ import org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver; import org.apache.druid.segment.realtime.appenderator.BatchAppenderatorDriver; import org.apache.druid.segment.realtime.appenderator.SegmentAllocator; -import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.segment.realtime.appenderator.SegmentsAndMetadata; import org.apache.druid.segment.realtime.appenderator.TransactionalSegmentPublisher; import org.apache.druid.segment.realtime.firehose.ChatHandler; @@ -88,12 +86,12 @@ import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec; +import org.apache.druid.timeline.partition.HashBasedNumberedShardSpecFactory; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.apache.druid.utils.CircularBuffer; -import org.apache.druid.utils.CollectionUtils; import org.codehaus.plexus.util.FileUtils; -import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; @@ -108,24 +106,20 @@ import javax.ws.rs.core.Response; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.SortedSet; import java.util.TreeMap; -import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -public class IndexTask extends AbstractTask implements ChatHandler +public class IndexTask extends AbstractBatchIndexTask implements ChatHandler { private static final Logger log = new Logger(IndexTask.class); private static final HashFunction hashFunction = Hashing.murmur3_128(); @@ -160,22 +154,22 @@ private static String makeGroupId(boolean isAppendToExisting, String dataSource) private final Optional chatHandlerProvider; @JsonIgnore - private FireDepartmentMetrics buildSegmentsFireDepartmentMetrics; + private final RowIngestionMeters determinePartitionsMeters; @JsonIgnore - private CircularBuffer buildSegmentsSavedParseExceptions; + private final RowIngestionMeters buildSegmentsMeters; @JsonIgnore - private CircularBuffer determinePartitionsSavedParseExceptions; + private FireDepartmentMetrics buildSegmentsFireDepartmentMetrics; @JsonIgnore - private String errorMsg; + private CircularBuffer buildSegmentsSavedParseExceptions; @JsonIgnore - private final RowIngestionMeters determinePartitionsMeters; + private CircularBuffer determinePartitionsSavedParseExceptions; @JsonIgnore - private final RowIngestionMeters buildSegmentsMeters; + private String errorMsg; @JsonCreator public IndexTask( @@ -252,36 +246,44 @@ public String getType() @Override public boolean isReady(TaskActionClient taskActionClient) throws Exception { - final Optional> intervals = ingestionSchema.getDataSchema() - .getGranularitySpec() - .bucketIntervals(); + return determineLockGranularityAndTryLock(taskActionClient, ingestionSchema.dataSchema.getGranularitySpec()); + } - if (intervals.isPresent()) { - return isReady(taskActionClient, intervals.get()); - } else { - return true; - } + @Override + public boolean requireLockExistingSegments() + { + return isGuaranteedRollup(ingestionSchema.ioConfig, ingestionSchema.tuningConfig) + || !ingestionSchema.ioConfig.isAppendToExisting(); } - static boolean isReady(TaskActionClient actionClient, SortedSet intervals) throws IOException + @Override + public List findSegmentsToLock(TaskActionClient taskActionClient, List intervals) + throws IOException { - // Sanity check preventing empty intervals (which cannot be locked, and don't make sense anyway). - for (Interval interval : intervals) { - if (interval.toDurationMillis() == 0) { - throw new ISE("Cannot run with empty interval[%s]", interval); - } - } + return findInputSegments( + getDataSource(), + taskActionClient, + intervals, + ingestionSchema.ioConfig.firehoseFactory + ); + } - final List locks = getTaskLocks(actionClient); - if (locks.size() == 0) { - try { - Tasks.tryAcquireExclusiveLocks(actionClient, intervals); - } - catch (Exception e) { - return false; - } + @Override + public boolean isPerfectRollup() + { + return isGuaranteedRollup(ingestionSchema.ioConfig, ingestionSchema.tuningConfig); + } + + @Nullable + @Override + public Granularity getSegmentGranularity() + { + final GranularitySpec granularitySpec = ingestionSchema.getDataSchema().getGranularitySpec(); + if (granularitySpec instanceof ArbitraryGranularitySpec) { + return null; + } else { + return granularitySpec.getSegmentGranularity(); } - return true; } @GET @@ -433,33 +435,32 @@ public TaskStatus run(final TaskToolbox toolbox) // Initialize maxRowsPerSegment and maxTotalRows lazily final IndexTuningConfig tuningConfig = ingestionSchema.tuningConfig; - @Nullable final Integer maxRowsPerSegment = getValidMaxRowsPerSegment(tuningConfig); - @Nullable final Long maxTotalRows = getValidMaxTotalRows(tuningConfig); - final ShardSpecs shardSpecs = determineShardSpecs(toolbox, firehoseFactory, firehoseTempDir, maxRowsPerSegment); + @Nullable + final Integer maxRowsPerSegment = getValidMaxRowsPerSegment(tuningConfig); + @Nullable + final Long maxTotalRows = getValidMaxTotalRows(tuningConfig); + // Spec for segment allocation. This is used only for perfect rollup mode. + // See createSegmentAllocator(). + final Map> allocateSpec = determineShardSpecs( + toolbox, + firehoseFactory, + firehoseTempDir, + maxRowsPerSegment + ); + + final List allocateIntervals = new ArrayList<>(allocateSpec.keySet()); final DataSchema dataSchema; - final Map versions; if (determineIntervals) { - final SortedSet intervals = new TreeSet<>(Comparators.intervalsByStartThenEnd()); - intervals.addAll(shardSpecs.getIntervals()); - final Map locks = Tasks.tryAcquireExclusiveLocks( - toolbox.getTaskActionClient(), - intervals - ); - versions = CollectionUtils.mapValues(locks, TaskLock::getVersion); + if (!determineLockGranularityandTryLock(toolbox.getTaskActionClient(), allocateIntervals)) { + throw new ISE("Failed to get locks for intervals[%s]", allocateIntervals); + } dataSchema = ingestionSchema.getDataSchema().withGranularitySpec( ingestionSchema.getDataSchema() .getGranularitySpec() - .withIntervals( - JodaUtils.condenseIntervals( - shardSpecs.getIntervals() - ) - ) + .withIntervals(JodaUtils.condenseIntervals(allocateIntervals)) ); } else { - versions = getTaskLocks(toolbox.getTaskActionClient()) - .stream() - .collect(Collectors.toMap(TaskLock::getInterval, TaskLock::getVersion)); dataSchema = ingestionSchema.getDataSchema(); } @@ -467,8 +468,7 @@ public TaskStatus run(final TaskToolbox toolbox) return generateAndPublishSegments( toolbox, dataSchema, - shardSpecs, - versions, + allocateSpec, firehoseFactory, firehoseTempDir, maxRowsPerSegment, @@ -539,15 +539,6 @@ private Map getTaskCompletionRowStats() return metrics; } - private static String findVersion(Map versions, Interval interval) - { - return versions.entrySet().stream() - .filter(entry -> entry.getKey().contains(interval)) - .map(Entry::getValue) - .findFirst() - .orElseThrow(() -> new ISE("Cannot find a version for interval[%s]", interval)); - } - private static boolean isGuaranteedRollup(IndexIOConfig ioConfig, IndexTuningConfig tuningConfig) { Preconditions.checkState( @@ -562,7 +553,7 @@ private static boolean isGuaranteedRollup(IndexIOConfig ioConfig, IndexTuningCon * shardSpecs by itself. Intervals must be determined if they are not specified in {@link GranularitySpec}. * ShardSpecs must be determined if the perfect rollup must be guaranteed even though the number of shards is not * specified in {@link IndexTuningConfig}. - *

+ *

* If both intervals and shardSpecs don't have to be determined, this method simply returns {@link ShardSpecs} for the * given intervals. Here, if {@link IndexTuningConfig#numShards} is not specified, {@link NumberedShardSpec} is used. *

@@ -570,9 +561,9 @@ private static boolean isGuaranteedRollup(IndexIOConfig ioConfig, IndexTuningCon * them. If the perfect rollup must be guaranteed, {@link HashBasedNumberedShardSpec} is used for hash partitioning * of input data. In the future we may want to also support single-dimension partitioning. * - * @return generated {@link ShardSpecs} representing a map of intervals and corresponding shard specs + * @return a map indicating how many shardSpecs need to be created per interval. */ - private ShardSpecs determineShardSpecs( + private Map> determineShardSpecs( final TaskToolbox toolbox, final FirehoseFactory firehoseFactory, final File firehoseTempDir, @@ -595,12 +586,7 @@ private ShardSpecs determineShardSpecs( // if we were given number of shards per interval and the intervals, we don't need to scan the data if (!determineNumPartitions && !determineIntervals) { log.info("Skipping determine partition scan"); - return createShardSpecWithoutInputScan( - jsonMapper, - granularitySpec, - ioConfig, - tuningConfig - ); + return createShardSpecWithoutInputScan(granularitySpec, ioConfig, tuningConfig); } else { // determine intervals containing data and prime HLL collectors return createShardSpecsFromInput( @@ -617,43 +603,34 @@ private ShardSpecs determineShardSpecs( } } - private static ShardSpecs createShardSpecWithoutInputScan( - ObjectMapper jsonMapper, + private Map> createShardSpecWithoutInputScan( GranularitySpec granularitySpec, IndexIOConfig ioConfig, IndexTuningConfig tuningConfig ) { - final Map> shardSpecs = new HashMap<>(); + final Map> allocateSpec = new HashMap<>(); final SortedSet intervals = granularitySpec.bucketIntervals().get(); if (isGuaranteedRollup(ioConfig, tuningConfig)) { // Overwrite mode, guaranteed rollup: shardSpecs must be known in advance. final int numShards = tuningConfig.getNumShards() == null ? 1 : tuningConfig.getNumShards(); - for (Interval interval : intervals) { - final List intervalShardSpecs = IntStream.range(0, numShards) - .mapToObj( - shardId -> new HashBasedNumberedShardSpec( - shardId, - numShards, - tuningConfig.partitionDimensions, - jsonMapper - ) - ) - .collect(Collectors.toList()); - shardSpecs.put(interval, intervalShardSpecs); + allocateSpec.put( + interval, + createShardSpecFactoryForGuaranteedRollup(numShards, tuningConfig.partitionDimensions) + ); } } else { for (Interval interval : intervals) { - shardSpecs.put(interval, ImmutableList.of()); + allocateSpec.put(interval, null); } } - return new ShardSpecs(shardSpecs); + return allocateSpec; } - private ShardSpecs createShardSpecsFromInput( + private Map> createShardSpecsFromInput( ObjectMapper jsonMapper, IndexIngestionSpec ingestionSchema, FirehoseFactory firehoseFactory, @@ -678,7 +655,7 @@ private ShardSpecs createShardSpecsFromInput( determineNumPartitions ); - final Map> intervalToShardSpecs = new HashMap<>(); + final Map> allocateSpecs = new HashMap<>(); final int defaultNumShards = tuningConfig.getNumShards() == null ? 1 : tuningConfig.getNumShards(); for (final Map.Entry> entry : hllCollectors.entrySet()) { final Interval interval = entry.getKey(); @@ -697,25 +674,26 @@ private ShardSpecs createShardSpecsFromInput( } if (isGuaranteedRollup(ingestionSchema.getIOConfig(), ingestionSchema.getTuningConfig())) { - // Overwrite mode, guaranteed rollup: shardSpecs must be known in advance. - final List intervalShardSpecs = IntStream.range(0, numShards) - .mapToObj( - shardId -> new HashBasedNumberedShardSpec( - shardId, - numShards, - tuningConfig.partitionDimensions, - jsonMapper - ) - ).collect(Collectors.toList()); - - intervalToShardSpecs.put(interval, intervalShardSpecs); + // Overwrite mode, guaranteed rollup: # of shards must be known in advance. + allocateSpecs.put( + interval, + createShardSpecFactoryForGuaranteedRollup(numShards, tuningConfig.partitionDimensions) + ); } else { - intervalToShardSpecs.put(interval, ImmutableList.of()); + allocateSpecs.put(interval, null); } } log.info("Found intervals and shardSpecs in %,dms", System.currentTimeMillis() - determineShardSpecsStartMillis); - return new ShardSpecs(intervalToShardSpecs); + return allocateSpecs; + } + + private Pair createShardSpecFactoryForGuaranteedRollup( + int numShards, + @Nullable List partitionDimensions + ) + { + return Pair.of(new HashBasedNumberedShardSpecFactory(partitionDimensions, numShards), numShards); } private Map> collectIntervalsAndShardSpecs( @@ -811,6 +789,32 @@ private Map> collectIntervalsAndShardSp return hllCollectors; } + private IndexTaskSegmentAllocator createSegmentAllocator( + TaskToolbox toolbox, + DataSchema dataSchema, + Map> allocateSpec + ) throws IOException + { + if (ingestionSchema.ioConfig.isAppendToExisting() || isUseSegmentLock()) { + return new RemoteSegmentAllocator( + toolbox, + getId(), + dataSchema, + getSegmentLockHelper(), + isUseSegmentLock() ? LockGranularity.SEGMENT : LockGranularity.TIME_CHUNK, + ingestionSchema.ioConfig.isAppendToExisting() + ); + } else { + // We use the timeChunk lock and don't have to ask the overlord to create segmentIds. + // Instead, a local allocator is used. + if (isGuaranteedRollup(ingestionSchema.ioConfig, ingestionSchema.tuningConfig)) { + return new CachingLocalSegmentAllocator(toolbox, getId(), getDataSource(), allocateSpec); + } else { + return new LocalSegmentAllocator(toolbox, getId(), getDataSource(), dataSchema.getGranularitySpec()); + } + } + } + /** * This method reads input data row by row and adds the read row to a proper segment using {@link BaseAppenderatorDriver}. * If there is no segment for the row, a new one is created. Segments can be published in the middle of reading inputs @@ -824,16 +828,15 @@ private Map> collectIntervalsAndShardSp * If the number of rows added to {@link BaseAppenderatorDriver} so far exceeds {@link IndexTuningConfig#maxTotalRows} * * - * + *

* At the end of this method, all the remaining segments are published. * - * @return true if generated segments are successfully published, otherwise false + * @return the last {@link TaskStatus} */ private TaskStatus generateAndPublishSegments( final TaskToolbox toolbox, final DataSchema dataSchema, - final ShardSpecs shardSpecs, - final Map versions, + final Map> allocateSpec, final FirehoseFactory firehoseFactory, final File firehoseTempDir, @Nullable final Integer maxRowsPerSegment, @@ -855,71 +858,19 @@ private TaskStatus generateAndPublishSegments( } final IndexIOConfig ioConfig = ingestionSchema.getIOConfig(); - final IndexTuningConfig tuningConfig = ingestionSchema.tuningConfig; + final IndexTuningConfig tuningConfig = ingestionSchema.getTuningConfig(); final long pushTimeout = tuningConfig.getPushTimeout(); final boolean isGuaranteedRollup = isGuaranteedRollup(ioConfig, tuningConfig); - final SegmentAllocator segmentAllocator; - if (isGuaranteedRollup) { - // Overwrite mode, guaranteed rollup: segments are all known in advance and there is one per sequenceName. - final Map lookup = new HashMap<>(); - - for (Map.Entry> entry : shardSpecs.getMap().entrySet()) { - for (ShardSpec shardSpec : entry.getValue()) { - final String version = findVersion(versions, entry.getKey()); - lookup.put( - Appenderators.getSequenceName(entry.getKey(), version, shardSpec), - new SegmentIdWithShardSpec(getDataSource(), entry.getKey(), version, shardSpec) - ); - } - } - - segmentAllocator = (row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> lookup.get(sequenceName); - } else if (ioConfig.isAppendToExisting()) { - // Append mode: Allocate segments as needed using Overlord APIs. - segmentAllocator = new ActionBasedSegmentAllocator( - toolbox.getTaskActionClient(), - dataSchema, - (schema, row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> new SegmentAllocateAction( - schema.getDataSource(), - row.getTimestamp(), - schema.getGranularitySpec().getQueryGranularity(), - schema.getGranularitySpec().getSegmentGranularity(), - sequenceName, - previousSegmentId, - skipSegmentLineageCheck - ) - ); - } else { - // Overwrite mode, non-guaranteed rollup: We can make up our own segment ids but we don't know them in advance. - final Map counters = new HashMap<>(); - - segmentAllocator = (row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> { - final DateTime timestamp = row.getTimestamp(); - Optional maybeInterval = granularitySpec.bucketInterval(timestamp); - if (!maybeInterval.isPresent()) { - throw new ISE("Could not find interval for timestamp [%s]", timestamp); - } - - final Interval interval = maybeInterval.get(); - if (!shardSpecs.getMap().containsKey(interval)) { - throw new ISE("Could not find shardSpec for interval[%s]", interval); - } - - final int partitionNum = counters.computeIfAbsent(interval, x -> new AtomicInteger()).getAndIncrement(); - return new SegmentIdWithShardSpec( - getDataSource(), - interval, - findVersion(versions, interval), - new NumberedShardSpec(partitionNum, 0) - ); - }; - } + final IndexTaskSegmentAllocator segmentAllocator = createSegmentAllocator( + toolbox, + dataSchema, + allocateSpec + ); - final TransactionalSegmentPublisher publisher = (segments, commitMetadata) -> { - final SegmentTransactionalInsertAction action = new SegmentTransactionalInsertAction(segments); - return toolbox.getTaskActionClient().submit(action); - }; + final TransactionalSegmentPublisher publisher = (segmentsToBeOverwritten, segmentsToPublish, commitMetadata) -> + toolbox.getTaskActionClient() + .submit(SegmentTransactionalInsertAction.overwriteAction(segmentsToBeOverwritten, segmentsToPublish)); try ( final Appenderator appenderator = newAppenderator( @@ -956,18 +907,8 @@ private TaskStatus generateAndPublishSegments( continue; } - final String sequenceName; - - if (isGuaranteedRollup) { - // Sequence name is based solely on the shardSpec, and there will only be one segment per sequence. - final Interval interval = optInterval.get(); - final ShardSpec shardSpec = shardSpecs.getShardSpec(interval, inputRow); - sequenceName = Appenderators.getSequenceName(interval, findVersion(versions, interval), shardSpec); - } else { - // Segments are created as needed, using a single sequence name. They may be allocated from the overlord - // (in append mode) or may be created on our own authority (in overwrite mode). - sequenceName = getId(); - } + final Interval interval = optInterval.get(); + final String sequenceName = segmentAllocator.getSequenceName(interval, inputRow); final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName); if (addResult.isOk()) { @@ -997,10 +938,13 @@ private TaskStatus generateAndPublishSegments( final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); log.info("Pushed segments[%s]", pushed.getSegments()); - final SegmentsAndMetadata published = awaitPublish( - driver.publishAll(publisher), - pushTimeout - ); + // If we use timeChunk lock, then we don't have to specify what segments will be overwritten because + // it will just overwrite all segments overlapped with the new segments. + final Set inputSegments = isUseSegmentLock() + ? getSegmentLockHelper().getLockedExistingSegments() + : null; + // Probably we can publish atomicUpdateGroup along with segments. + final SegmentsAndMetadata published = awaitPublish(driver.publishAll(inputSegments, publisher), pushTimeout); ingestionState = IngestionState.COMPLETED; if (published == null) { @@ -1036,8 +980,10 @@ private TaskStatus generateAndPublishSegments( */ public static Integer getValidMaxRowsPerSegment(IndexTuningConfig tuningConfig) { - @Nullable final Integer numShards = tuningConfig.numShards; - @Nullable final Integer maxRowsPerSegment = tuningConfig.maxRowsPerSegment; + @Nullable + final Integer numShards = tuningConfig.numShards; + @Nullable + final Integer maxRowsPerSegment = tuningConfig.maxRowsPerSegment; if (numShards == null || numShards == -1) { return maxRowsPerSegment == null || maxRowsPerSegment.equals(-1) ? IndexTuningConfig.DEFAULT_MAX_ROWS_PER_SEGMENT @@ -1054,8 +1000,10 @@ public static Integer getValidMaxRowsPerSegment(IndexTuningConfig tuningConfig) */ public static Long getValidMaxTotalRows(IndexTuningConfig tuningConfig) { - @Nullable final Integer numShards = tuningConfig.numShards; - @Nullable final Long maxTotalRows = tuningConfig.maxTotalRows; + @Nullable + final Integer numShards = tuningConfig.numShards; + @Nullable + final Long maxTotalRows = tuningConfig.maxTotalRows; if (numShards == null || numShards == -1) { return maxTotalRows == null ? IndexTuningConfig.DEFAULT_MAX_TOTAL_ROWS : maxTotalRows; } else { @@ -1378,8 +1326,8 @@ private IndexTuningConfig( ); this.maxRowsPerSegment = (maxRowsPerSegment != null && maxRowsPerSegment == -1) - ? null - : maxRowsPerSegment; + ? null + : maxRowsPerSegment; this.maxRowsInMemory = maxRowsInMemory == null ? TuningConfig.DEFAULT_MAX_ROWS_IN_MEMORY : maxRowsInMemory; // initializing this to 0, it will be lazily initialized to a value // @see server.src.main.java.org.apache.druid.segment.indexing.TuningConfigs#getMaxBytesInMemoryOrDefault(long) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTaskSegmentAllocator.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTaskSegmentAllocator.java new file mode 100644 index 000000000000..d997675f5962 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTaskSegmentAllocator.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.task; + +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.realtime.appenderator.SegmentAllocator; +import org.joda.time.Interval; + +/** + * Segment allocator interface for {@link IndexTask}. It has 3 different modes for allocating segments. + */ +interface IndexTaskSegmentAllocator extends SegmentAllocator +{ + /** + * SequenceName is the key to create the segmentId. If previousSegmentId is given, {@link SegmentAllocator} allocates + * segmentId depending on sequenceName and previousSegmentId. If it's missing, it allocates segmentId using + * sequenceName and interval. For {@link IndexTask}, it always provides the previousSegmentId to + * SegmentAllocator. + * See {@link org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator#allocatePendingSegment} for details. + * + * Implementations should return the correct sequenceName based on the given interval and inputRow, which is passed + * to SegmentAllocator. + */ + String getSequenceName(Interval interval, InputRow inputRow); +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/KillTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/KillTask.java index ac15e67e74b1..ea7ac46c984e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/KillTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/KillTask.java @@ -21,21 +21,27 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import org.apache.druid.client.indexing.ClientKillQuery; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.actions.SegmentListUnusedAction; import org.apache.druid.indexing.common.actions.SegmentNukeAction; +import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TaskLocks; import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.timeline.DataSegment; +import org.joda.time.DateTime; import org.joda.time.Interval; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.stream.Collectors; /** * The client representation of this task is {@link ClientKillQuery}. @@ -44,8 +50,6 @@ */ public class KillTask extends AbstractFixedIntervalTask { - private static final Logger log = new Logger(KillTask.class); - @JsonCreator public KillTask( @JsonProperty("id") String id, @@ -71,42 +75,36 @@ public String getType() @Override public TaskStatus run(TaskToolbox toolbox) throws Exception { - // Confirm we have a lock (will throw if there isn't exactly one element) - final TaskLock myLock = Iterables.getOnlyElement(getTaskLocks(toolbox.getTaskActionClient())); - - if (!myLock.getDataSource().equals(getDataSource())) { - throw new ISE("WTF?! Lock dataSource[%s] != task dataSource[%s]", myLock.getDataSource(), getDataSource()); - } - - if (!myLock.getInterval().equals(getInterval())) { - throw new ISE("WTF?! Lock interval[%s] != task interval[%s]", myLock.getInterval(), getInterval()); - } - + final NavigableMap> taskLockMap = getTaskLockMap(toolbox.getTaskActionClient()); // List unused segments final List unusedSegments = toolbox .getTaskActionClient() - .submit(new SegmentListUnusedAction(myLock.getDataSource(), myLock.getInterval())); + .submit(new SegmentListUnusedAction(getDataSource(), getInterval())); - // Verify none of these segments have versions > lock version - for (final DataSegment unusedSegment : unusedSegments) { - if (unusedSegment.getVersion().compareTo(myLock.getVersion()) > 0) { - throw new ISE( - "WTF?! Unused segment[%s] has version[%s] > task version[%s]", - unusedSegment.getId(), - unusedSegment.getVersion(), - myLock.getVersion() - ); - } - - log.info("OK to kill segment: %s", unusedSegment.getId()); + if (!TaskLocks.isLockCoversSegments(taskLockMap, unusedSegments)) { + throw new ISE( + "Locks[%s] for task[%s] can't cover segments[%s]", + taskLockMap.values().stream().flatMap(List::stream).collect(Collectors.toList()), + getId(), + unusedSegments + ); } // Kill segments + toolbox.getTaskActionClient().submit(new SegmentNukeAction(new HashSet<>(unusedSegments))); for (DataSegment segment : unusedSegments) { toolbox.getDataSegmentKiller().kill(segment); - toolbox.getTaskActionClient().submit(new SegmentNukeAction(ImmutableSet.of(segment))); } return TaskStatus.success(getId()); } + + private NavigableMap> getTaskLockMap(TaskActionClient client) throws IOException + { + final NavigableMap> taskLockMap = new TreeMap<>(); + getTaskLocks(client).forEach( + taskLock -> taskLockMap.computeIfAbsent(taskLock.getInterval().getStart(), k -> new ArrayList<>()).add(taskLock) + ); + return taskLockMap; + } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/LocalSegmentAllocator.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/LocalSegmentAllocator.java new file mode 100644 index 000000000000..3c8e99ab536d --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/LocalSegmentAllocator.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.task; + +import com.google.common.base.Optional; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskToolbox; +import org.apache.druid.indexing.common.actions.LockListAction; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.segment.indexing.granularity.GranularitySpec; +import org.apache.druid.segment.realtime.appenderator.SegmentAllocator; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.joda.time.DateTime; +import org.joda.time.Interval; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +/** + * Segment allocator which allocates new segments locally per request. + */ +class LocalSegmentAllocator implements IndexTaskSegmentAllocator +{ + private final String taskId; + + private final SegmentAllocator internalAllocator; + + LocalSegmentAllocator(TaskToolbox toolbox, String taskId, String dataSource, GranularitySpec granularitySpec) + throws IOException + { + this.taskId = taskId; + final Map counters = new HashMap<>(); + + final Map intervalToVersion = toolbox.getTaskActionClient() + .submit(new LockListAction()) + .stream() + .collect(Collectors.toMap(TaskLock::getInterval, TaskLock::getVersion)); + + internalAllocator = (row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> { + final DateTime timestamp = row.getTimestamp(); + Optional maybeInterval = granularitySpec.bucketInterval(timestamp); + if (!maybeInterval.isPresent()) { + throw new ISE("Could not find interval for timestamp [%s]", timestamp); + } + + final Interval interval = maybeInterval.get(); + final String version = intervalToVersion.entrySet().stream() + .filter(entry -> entry.getKey().contains(interval)) + .map(Entry::getValue) + .findFirst() + .orElseThrow(() -> new ISE("Cannot find a version for interval[%s]", interval)); + + final int partitionNum = counters.computeIfAbsent(interval, x -> new AtomicInteger()).getAndIncrement(); + return new SegmentIdWithShardSpec( + dataSource, + interval, + version, + new NumberedShardSpec(partitionNum, 0) + ); + }; + } + + @Override + public String getSequenceName(Interval interval, InputRow inputRow) + { + // Segments are created as needed, using a single sequence name. They may be allocated from the overlord + // (in append mode) or may be created on our own authority (in overwrite mode). + return taskId; + } + + @Override + public SegmentIdWithShardSpec allocate( + InputRow row, + String sequenceName, + String previousSegmentId, + boolean skipSegmentLineageCheck + ) throws IOException + { + return internalAllocator.allocate(row, sequenceName, previousSegmentId, skipSegmentLineageCheck); + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/NoopTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/NoopTask.java index cc9ed629be31..b400c3bd3bdb 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/NoopTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/NoopTask.java @@ -68,6 +68,7 @@ enum IsReadyResult @JsonCreator public NoopTask( @JsonProperty("id") String id, + @JsonProperty("groupId") String groupId, @JsonProperty("dataSource") String dataSource, @JsonProperty("runTime") long runTime, @JsonProperty("isReadyTime") long isReadyTime, @@ -78,6 +79,8 @@ public NoopTask( { super( id == null ? StringUtils.format("noop_%s_%s", DateTimes.nowUtc(), UUID.randomUUID().toString()) : id, + groupId, + null, dataSource == null ? "none" : dataSource, context ); @@ -159,24 +162,29 @@ public int getPriority() public static NoopTask create() { - return new NoopTask(null, null, 0, 0, null, null, null); + return new NoopTask(null, null, null, 0, 0, null, null, null); + } + + public static NoopTask withGroupId(String groupId) + { + return new NoopTask(null, groupId, null, 0, 0, null, null, null); } @VisibleForTesting public static NoopTask create(String dataSource) { - return new NoopTask(null, dataSource, 0, 0, null, null, null); + return new NoopTask(null, null, dataSource, 0, 0, null, null, null); } @VisibleForTesting public static NoopTask create(int priority) { - return new NoopTask(null, null, 0, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, priority)); + return new NoopTask(null, null, null, 0, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, priority)); } @VisibleForTesting public static NoopTask create(String id, int priority) { - return new NoopTask(id, null, 0, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, priority)); + return new NoopTask(id, null, null, 0, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, priority)); } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/RealtimeIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/RealtimeIndexTask.java index 2786805b9f4c..ce2b1c6cb65a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/RealtimeIndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/RealtimeIndexTask.java @@ -38,9 +38,9 @@ import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskRealtimeMetricsMonitorBuilder; import org.apache.druid.indexing.common.TaskToolbox; -import org.apache.druid.indexing.common.actions.LockAcquireAction; import org.apache.druid.indexing.common.actions.LockReleaseAction; import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TimeChunkLockAcquireAction; import org.apache.druid.indexing.common.config.TaskConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.StringUtils; @@ -240,7 +240,7 @@ public void announceSegment(final DataSegment segment) throws IOException // Side effect: Calling announceSegment causes a lock to be acquired Preconditions.checkNotNull( toolbox.getTaskActionClient().submit( - new LockAcquireAction(TaskLockType.EXCLUSIVE, segment.getInterval(), lockTimeoutMs) + new TimeChunkLockAcquireAction(TaskLockType.EXCLUSIVE, segment.getInterval(), lockTimeoutMs) ), "Cannot acquire a lock for interval[%s]", segment.getInterval() @@ -266,7 +266,7 @@ public void announceSegments(Iterable segments) throws IOException for (DataSegment segment : segments) { Preconditions.checkNotNull( toolbox.getTaskActionClient().submit( - new LockAcquireAction(TaskLockType.EXCLUSIVE, segment.getInterval(), lockTimeoutMs) + new TimeChunkLockAcquireAction(TaskLockType.EXCLUSIVE, segment.getInterval(), lockTimeoutMs) ), "Cannot acquire a lock for interval[%s]", segment.getInterval() @@ -302,7 +302,7 @@ public String getVersion(final Interval interval) { try { // Side effect: Calling getVersion causes a lock to be acquired - final LockAcquireAction action = new LockAcquireAction(TaskLockType.EXCLUSIVE, interval, lockTimeoutMs); + final TimeChunkLockAcquireAction action = new TimeChunkLockAcquireAction(TaskLockType.EXCLUSIVE, interval, lockTimeoutMs); final TaskLock lock = Preconditions.checkNotNull( toolbox.getTaskActionClient().submit(action), "Cannot acquire a lock for interval[%s]", diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/RemoteSegmentAllocator.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/RemoteSegmentAllocator.java new file mode 100644 index 000000000000..4fc117df88fb --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/RemoteSegmentAllocator.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.task; + +import org.apache.druid.data.input.InputRow; +import org.apache.druid.indexing.appenderator.ActionBasedSegmentAllocator; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.TaskToolbox; +import org.apache.druid.indexing.common.actions.SegmentAllocateAction; +import org.apache.druid.indexing.common.task.SegmentLockHelper.OverwritingRootGenerationPartitions; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.segment.indexing.DataSchema; +import org.apache.druid.segment.indexing.granularity.GranularitySpec; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; +import org.apache.druid.timeline.partition.NumberedOverwritingShardSpecFactory; +import org.apache.druid.timeline.partition.NumberedShardSpecFactory; +import org.apache.druid.timeline.partition.ShardSpecFactory; +import org.joda.time.Interval; + +import java.io.IOException; + +/** + * Segment allocator which allocates new segments using the overlord per request. + */ +public class RemoteSegmentAllocator implements IndexTaskSegmentAllocator +{ + private final String taskId; + private final ActionBasedSegmentAllocator internalAllocator; + + RemoteSegmentAllocator( + final TaskToolbox toolbox, + final String taskId, + final DataSchema dataSchema, + final SegmentLockHelper segmentLockHelper, + final LockGranularity lockGranularity, + final boolean appendToExisting + ) + { + this.taskId = taskId; + this.internalAllocator = new ActionBasedSegmentAllocator( + toolbox.getTaskActionClient(), + dataSchema, + (schema, row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> { + final GranularitySpec granularitySpec = schema.getGranularitySpec(); + final Interval interval = granularitySpec + .bucketInterval(row.getTimestamp()) + .or(granularitySpec.getSegmentGranularity().bucket(row.getTimestamp())); + if (lockGranularity == LockGranularity.TIME_CHUNK) { + return new SegmentAllocateAction( + schema.getDataSource(), + row.getTimestamp(), + schema.getGranularitySpec().getQueryGranularity(), + schema.getGranularitySpec().getSegmentGranularity(), + sequenceName, + previousSegmentId, + skipSegmentLineageCheck, + NumberedShardSpecFactory.instance(), + lockGranularity + ); + } else { + final ShardSpecFactory shardSpecFactory; + if (segmentLockHelper.hasLockedExistingSegments() && !appendToExisting) { + final OverwritingRootGenerationPartitions overwritingSegmentMeta = segmentLockHelper + .getOverwritingRootGenerationPartition(interval); + if (overwritingSegmentMeta == null) { + throw new ISE("Can't find overwritingSegmentMeta for interval[%s]", interval); + } + shardSpecFactory = new NumberedOverwritingShardSpecFactory( + overwritingSegmentMeta.getStartRootPartitionId(), + overwritingSegmentMeta.getEndRootPartitionId(), + overwritingSegmentMeta.getMinorVersionForNewSegments() + ); + } else { + shardSpecFactory = NumberedShardSpecFactory.instance(); + } + return new SegmentAllocateAction( + schema.getDataSource(), + row.getTimestamp(), + schema.getGranularitySpec().getQueryGranularity(), + schema.getGranularitySpec().getSegmentGranularity(), + sequenceName, + previousSegmentId, + skipSegmentLineageCheck, + shardSpecFactory, + lockGranularity + ); + } + } + ); + } + + @Override + public SegmentIdWithShardSpec allocate( + InputRow row, + String sequenceName, + String previousSegmentId, + boolean skipSegmentLineageCheck + ) throws IOException + { + return internalAllocator.allocate(row, sequenceName, previousSegmentId, skipSegmentLineageCheck); + } + + @Override + public String getSequenceName(Interval interval, InputRow inputRow) + { + // Segments are created as needed, using a single sequence name. They may be allocated from the overlord + // (in append mode) or may be created on our own authority (in overwrite mode). + return taskId; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/SegmentLockHelper.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/SegmentLockHelper.java new file mode 100644 index 000000000000..020f29efa3ae --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/SegmentLockHelper.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.task; + +import com.google.common.base.Preconditions; +import org.apache.druid.indexing.common.SegmentLock; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.actions.SegmentLockReleaseAction; +import org.apache.druid.indexing.common.actions.SegmentLockTryAcquireAction; +import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.overlord.LockResult; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.timeline.DataSegment; +import org.joda.time.Interval; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * This class provides some methods to use the segment lock easier and caches the information of locked segments. + */ +public class SegmentLockHelper +{ + private final Map overwritingRootGenPartitions = new HashMap<>(); + private final Set lockedExistingSegments = new HashSet<>(); + @Nullable + private Granularity knownSegmentGranularity; + + public static class OverwritingRootGenerationPartitions + { + private final int startRootPartitionId; + private final int endRootPartitionId; + private final short maxMinorVersion; + + private OverwritingRootGenerationPartitions(int startRootPartitionId, int endRootPartitionId, short maxMinorVersion) + { + this.startRootPartitionId = startRootPartitionId; + this.endRootPartitionId = endRootPartitionId; + this.maxMinorVersion = maxMinorVersion; + } + + public int getStartRootPartitionId() + { + return startRootPartitionId; + } + + public int getEndRootPartitionId() + { + return endRootPartitionId; + } + + public short getMinorVersionForNewSegments() + { + return (short) (maxMinorVersion + 1); + } + } + + public boolean hasLockedExistingSegments() + { + return !lockedExistingSegments.isEmpty(); + } + + public boolean hasOverwritingRootGenerationPartition(Interval interval) + { + return overwritingRootGenPartitions.containsKey(interval); + } + + public Set getLockedExistingSegments() + { + return Collections.unmodifiableSet(lockedExistingSegments); + } + + public OverwritingRootGenerationPartitions getOverwritingRootGenerationPartition(Interval interval) + { + return overwritingRootGenPartitions.get(interval); + } + + boolean verifyAndLockExistingSegments(TaskActionClient actionClient, List segments) + throws IOException + { + final List segmentsToLock = segments.stream() + .filter(segment -> !lockedExistingSegments.contains(segment)) + .collect(Collectors.toList()); + if (segmentsToLock.isEmpty()) { + return true; + } + + verifySegmentGranularity(segmentsToLock); + return tryLockSegments(actionClient, segmentsToLock); + } + + /** + * Check if segmentGranularity has changed. + */ + private void verifySegmentGranularity(List segments) + { + final Granularity granularityFromSegments = AbstractBatchIndexTask.findGranularityFromSegments(segments); + if (granularityFromSegments != null) { + if (knownSegmentGranularity == null) { + knownSegmentGranularity = granularityFromSegments; + } else { + if (!knownSegmentGranularity.equals(granularityFromSegments)) { + throw new ISE( + "Found a different granularity from knownSegmentGranularity[%s] in segments[%s]", + knownSegmentGranularity, + segments + ); + } + final List nonAlignedSegments = segments + .stream() + .filter(segment -> !knownSegmentGranularity.isAligned(segment.getInterval())) + .collect(Collectors.toList()); + + if (!nonAlignedSegments.isEmpty()) { + throw new ISE( + "Non-aligned segments[%s] for granularity[%s]", + nonAlignedSegments.stream().map(DataSegment::getId).collect(Collectors.toList()), + knownSegmentGranularity + ); + } + } + } else { + throw new ISE( + "Found different granularities in segments[%s]", + segments.stream().map(DataSegment::getId).collect(Collectors.toList()) + ); + } + } + + private boolean tryLockSegments(TaskActionClient actionClient, List segments) throws IOException + { + final Map> intervalToSegments = groupSegmentsByInterval(segments); + final Closer lockCloserOnError = Closer.create(); + for (Entry> entry : intervalToSegments.entrySet()) { + final Interval interval = entry.getKey(); + final List segmentsInInterval = entry.getValue(); + final boolean hasSameVersion = segmentsInInterval + .stream() + .allMatch(segment -> segment.getVersion().equals(segmentsInInterval.get(0).getVersion())); + Preconditions.checkState( + hasSameVersion, + "Segments[%s] should have same version", + segmentsInInterval.stream().map(DataSegment::getId).collect(Collectors.toList()) + ); + final List lockResults = actionClient.submit( + new SegmentLockTryAcquireAction( + TaskLockType.EXCLUSIVE, + interval, + segmentsInInterval.get(0).getVersion(), + segmentsInInterval.stream() + .map(segment -> segment.getShardSpec().getPartitionNum()) + .collect(Collectors.toSet()) + ) + ); + + lockResults.stream() + .filter(LockResult::isOk) + .map(result -> (SegmentLock) result.getTaskLock()) + .forEach(segmentLock -> lockCloserOnError.register(() -> actionClient.submit( + new SegmentLockReleaseAction(segmentLock.getInterval(), segmentLock.getPartitionId()) + ))); + if (lockResults.stream().anyMatch(result -> !result.isOk())) { + lockCloserOnError.close(); + return false; + } + lockedExistingSegments.addAll(segmentsInInterval); + verifyAndFindRootPartitionRangeAndMinorVersion(segmentsInInterval); + } + return true; + } + + /** + * This method is called when the task overwrites existing segments with segment locks. It verifies the input segments + * can be locked together, so that output segments can overshadow existing ones properly. + *

+ * This method checks two things: + *

+ * - Are rootPartition range of inputSegments adjacent? Two rootPartition ranges are adjacent if they are consecutive. + * - All atomicUpdateGroups of inputSegments must be full. (See {@code AtomicUpdateGroup#isFull()}). + */ + private void verifyAndFindRootPartitionRangeAndMinorVersion(List inputSegments) + { + if (inputSegments.isEmpty()) { + return; + } + + final List sortedSegments = new ArrayList<>(inputSegments); + sortedSegments.sort((s1, s2) -> { + if (s1.getStartRootPartitionId() != s2.getStartRootPartitionId()) { + return Integer.compare(s1.getStartRootPartitionId(), s2.getStartRootPartitionId()); + } else { + return Integer.compare(s1.getEndRootPartitionId(), s2.getEndRootPartitionId()); + } + }); + verifyRootPartitionIsAdjacentAndAtomicUpdateGroupIsFull(sortedSegments); + final Interval interval = sortedSegments.get(0).getInterval(); + final short prevMaxMinorVersion = (short) sortedSegments + .stream() + .mapToInt(DataSegment::getMinorVersion) + .max() + .orElseThrow(() -> new ISE("Empty inputSegments")); + + overwritingRootGenPartitions.put( + interval, + new OverwritingRootGenerationPartitions( + sortedSegments.get(0).getStartRootPartitionId(), + sortedSegments.get(sortedSegments.size() - 1).getEndRootPartitionId(), + prevMaxMinorVersion + ) + ); + } + + public static void verifyRootPartitionIsAdjacentAndAtomicUpdateGroupIsFull(List sortedSegments) + { + if (sortedSegments.isEmpty()) { + return; + } + + Preconditions.checkArgument( + sortedSegments.stream().allMatch(segment -> segment.getInterval().equals(sortedSegments.get(0).getInterval())) + ); + + short atomicUpdateGroupSize = 1; + // sanity check + for (int i = 0; i < sortedSegments.size() - 1; i++) { + final DataSegment curSegment = sortedSegments.get(i); + final DataSegment nextSegment = sortedSegments.get(i + 1); + if (curSegment.getStartRootPartitionId() == nextSegment.getStartRootPartitionId() + && curSegment.getEndRootPartitionId() == nextSegment.getEndRootPartitionId()) { + // Input segments should have the same or consecutive rootPartition range + if (curSegment.getMinorVersion() != nextSegment.getMinorVersion() + || curSegment.getAtomicUpdateGroupSize() != nextSegment.getAtomicUpdateGroupSize()) { + throw new ISE( + "segment[%s] and segment[%s] have the same rootPartitionRange, but different minorVersion or atomicUpdateGroupSize", + curSegment, + nextSegment + ); + } + atomicUpdateGroupSize++; + } else { + if (curSegment.getEndRootPartitionId() != nextSegment.getStartRootPartitionId()) { + throw new ISE("Can't compact segments of non-consecutive rootPartition range"); + } + if (atomicUpdateGroupSize != curSegment.getAtomicUpdateGroupSize()) { + throw new ISE("All atomicUpdateGroup must be compacted together"); + } + atomicUpdateGroupSize = 1; + } + } + if (atomicUpdateGroupSize != sortedSegments.get(sortedSegments.size() - 1).getAtomicUpdateGroupSize()) { + throw new ISE("All atomicUpdateGroup must be compacted together"); + } + } + + private static Map> groupSegmentsByInterval(List segments) + { + final Map> map = new HashMap<>(); + for (DataSegment segment : segments) { + map.computeIfAbsent(segment.getInterval(), k -> new ArrayList<>()).add(segment); + } + return map; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Tasks.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Tasks.java index deebf3b38af5..3aa617d16faa 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Tasks.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Tasks.java @@ -19,20 +19,12 @@ package org.apache.druid.indexing.common.task; -import com.google.common.base.Preconditions; -import org.apache.druid.indexing.common.TaskLock; -import org.apache.druid.indexing.common.TaskLockType; -import org.apache.druid.indexing.common.actions.LockTryAcquireAction; -import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.java.util.common.guava.Comparators; import org.joda.time.Interval; -import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.TimeUnit; @@ -44,23 +36,11 @@ public class Tasks public static final int DEFAULT_MERGE_TASK_PRIORITY = 25; public static final int DEFAULT_TASK_PRIORITY = 0; public static final long DEFAULT_LOCK_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); + public static final boolean DEFAULT_FORCE_TIME_CHUNK_LOCK = true; public static final String PRIORITY_KEY = "priority"; public static final String LOCK_TIMEOUT_KEY = "taskLockTimeout"; - - public static Map tryAcquireExclusiveLocks(TaskActionClient client, SortedSet intervals) - throws IOException - { - final Map lockMap = new HashMap<>(); - for (Interval interval : computeCompactIntervals(intervals)) { - final TaskLock lock = Preconditions.checkNotNull( - client.submit(new LockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)), - "Cannot acquire a lock for interval[%s]", interval - ); - lockMap.put(interval, lock); - } - return lockMap; - } + public static final String FORCE_TIME_CHUNK_LOCK_KEY = "forceTimeChunkLock"; public static SortedSet computeCompactIntervals(SortedSet intervals) { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSubTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSubTask.java index cf53f15f4ccf..8d98df8358b5 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSubTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSubTask.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; -import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; import org.apache.druid.client.indexing.IndexingServiceClient; import org.apache.druid.data.input.Firehose; @@ -32,27 +32,32 @@ import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.appenderator.ActionBasedSegmentAllocator; import org.apache.druid.indexing.appenderator.ActionBasedUsedSegmentChecker; -import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.TaskToolbox; -import org.apache.druid.indexing.common.actions.LockTryAcquireAction; import org.apache.druid.indexing.common.actions.SegmentAllocateAction; import org.apache.druid.indexing.common.actions.SurrogateAction; import org.apache.druid.indexing.common.actions.TaskActionClient; -import org.apache.druid.indexing.common.task.AbstractTask; +import org.apache.druid.indexing.common.task.AbstractBatchIndexTask; import org.apache.druid.indexing.common.task.ClientBasedTaskInfoProvider; import org.apache.druid.indexing.common.task.IndexTask; import org.apache.druid.indexing.common.task.IndexTaskClientFactory; +import org.apache.druid.indexing.common.task.SegmentLockHelper; +import org.apache.druid.indexing.common.task.SegmentLockHelper.OverwritingRootGenerationPartitions; import org.apache.druid.indexing.common.task.TaskResource; import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.ParseException; import org.apache.druid.query.DruidMetrics; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.RealtimeIOConfig; +import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import org.apache.druid.segment.indexing.granularity.GranularitySpec; +import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; import org.apache.druid.segment.realtime.FireDepartment; import org.apache.druid.segment.realtime.FireDepartmentMetrics; import org.apache.druid.segment.realtime.RealtimeMetricsMonitor; @@ -62,27 +67,34 @@ import org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver; import org.apache.druid.segment.realtime.appenderator.BatchAppenderatorDriver; import org.apache.druid.segment.realtime.appenderator.SegmentAllocator; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.segment.realtime.appenderator.SegmentsAndMetadata; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.NumberedOverwritingShardSpecFactory; +import org.apache.druid.timeline.partition.NumberedShardSpecFactory; +import org.apache.druid.timeline.partition.PartitionChunk; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.joda.time.Interval; import javax.annotation.Nullable; import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.SortedSet; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; /** * A worker task of {@link ParallelIndexSupervisorTask}. Similar to {@link IndexTask}, but this task * generates and pushes segments, and reports them to the {@link ParallelIndexSupervisorTask} instead of * publishing on its own. */ -public class ParallelIndexSubTask extends AbstractTask +public class ParallelIndexSubTask extends AbstractBatchIndexTask { public static final String TYPE = "index_sub"; @@ -140,25 +152,20 @@ public String getType() } @Override - public boolean isReady(TaskActionClient taskActionClient) + public boolean isReady(TaskActionClient taskActionClient) throws IOException { - final Optional> intervals = ingestionSchema.getDataSchema() - .getGranularitySpec() - .bucketIntervals(); - - return !intervals.isPresent() || checkLockAcquired(taskActionClient, intervals.get()); - } - - private boolean checkLockAcquired(TaskActionClient actionClient, SortedSet intervals) - { - try { - tryAcquireExclusiveSurrogateLocks(actionClient, intervals); - return true; - } - catch (Exception e) { - log.error(e, "Failed to acquire locks for intervals[%s]", intervals); - return false; + if (!ingestionSchema.getDataSchema().getGranularitySpec().bucketIntervals().isPresent() + && !ingestionSchema.getIOConfig().isAppendToExisting()) { + // If intervals are missing in the granularitySpec, parallel index task runs in "dynamic locking mode". + // In this mode, sub tasks ask new locks whenever they see a new row which is not covered by existing locks. + // If this task is overwriting existing segments, then we should know this task is changing segment granularity + // in advance to know what types of lock we should use. However, if intervals are missing, we can't know + // the segment granularity of existing segments until the task reads all data because we don't know what segments + // are going to be overwritten. As a result, we assume that segment granularity will be changed if intervals are + // missing force to use timeChunk lock. + addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, true); } + return determineLockGranularityAndTryLock(taskActionClient, ingestionSchema.getDataSchema().getGranularitySpec()); } @JsonProperty @@ -195,63 +202,147 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception ingestionSchema.getTuningConfig().getChatHandlerTimeout(), ingestionSchema.getTuningConfig().getChatHandlerNumRetries() ); - final List pushedSegments = generateAndPushSegments( + final Set pushedSegments = generateAndPushSegments( toolbox, taskClient, firehoseFactory, firehoseTempDir ); - taskClient.report(supervisorTaskId, pushedSegments); + + // Find inputSegments overshadowed by pushedSegments + final Set allSegments = new HashSet<>(getSegmentLockHelper().getLockedExistingSegments()); + allSegments.addAll(pushedSegments); + final VersionedIntervalTimeline timeline = VersionedIntervalTimeline.forSegments(allSegments); + final Set oldSegments = timeline.findFullyOvershadowed() + .stream() + .flatMap(holder -> holder.getObject().stream()) + .map(PartitionChunk::getObject) + .collect(Collectors.toSet()); + taskClient.report(supervisorTaskId, oldSegments, pushedSegments); return TaskStatus.success(getId()); } - private void tryAcquireExclusiveSurrogateLocks( - TaskActionClient client, - SortedSet intervals - ) + @Override + public boolean requireLockExistingSegments() + { + return !ingestionSchema.getIOConfig().isAppendToExisting(); + } + + @Override + public List findSegmentsToLock(TaskActionClient taskActionClient, List intervals) throws IOException { - for (Interval interval : Tasks.computeCompactIntervals(intervals)) { - Preconditions.checkNotNull( - client.submit( - new SurrogateAction<>(supervisorTaskId, new LockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)) - ), - "Cannot acquire a lock for interval[%s]", interval - ); - } + return findInputSegments( + getDataSource(), + taskActionClient, + intervals, + ingestionSchema.getIOConfig().getFirehoseFactory() + ); } - private SegmentAllocator createSegmentAllocator( - TaskToolbox toolbox, - ParallelIndexTaskClient taskClient, - ParallelIndexIngestionSpec ingestionSchema - ) + @Override + public boolean isPerfectRollup() { - final DataSchema dataSchema = ingestionSchema.getDataSchema(); - final ParallelIndexIOConfig ioConfig = ingestionSchema.getIOConfig(); - if (ioConfig.isAppendToExisting()) { - return new ActionBasedSegmentAllocator( - toolbox.getTaskActionClient(), - dataSchema, - (schema, row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> new SurrogateAction<>( - supervisorTaskId, - new SegmentAllocateAction( - schema.getDataSource(), - row.getTimestamp(), - schema.getGranularitySpec().getQueryGranularity(), - schema.getGranularitySpec().getSegmentGranularity(), - sequenceName, - previousSegmentId, - skipSegmentLineageCheck - ) - ) - ); + return false; + } + + @Nullable + @Override + public Granularity getSegmentGranularity() + { + final GranularitySpec granularitySpec = ingestionSchema.getDataSchema().getGranularitySpec(); + if (granularitySpec instanceof ArbitraryGranularitySpec) { + return null; } else { - return (row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> taskClient.allocateSegment( - supervisorTaskId, - row.getTimestamp() - ); + return granularitySpec.getSegmentGranularity(); + } + } + + @VisibleForTesting + SegmentAllocator createSegmentAllocator(TaskToolbox toolbox, ParallelIndexTaskClient taskClient) + { + return new WrappingSegmentAllocator(toolbox, taskClient); + } + + private class WrappingSegmentAllocator implements SegmentAllocator + { + private final TaskToolbox toolbox; + private final ParallelIndexTaskClient taskClient; + + private SegmentAllocator internalAllocator; + + private WrappingSegmentAllocator(TaskToolbox toolbox, ParallelIndexTaskClient taskClient) + { + this.toolbox = toolbox; + this.taskClient = taskClient; + } + + @Override + public SegmentIdWithShardSpec allocate( + InputRow row, + String sequenceName, + String previousSegmentId, + boolean skipSegmentLineageCheck + ) throws IOException + { + if (internalAllocator == null) { + internalAllocator = createSegmentAllocator(); + } + return internalAllocator.allocate(row, sequenceName, previousSegmentId, skipSegmentLineageCheck); + } + + private SegmentAllocator createSegmentAllocator() + { + final GranularitySpec granularitySpec = ingestionSchema.getDataSchema().getGranularitySpec(); + final SegmentLockHelper segmentLockHelper = getSegmentLockHelper(); + if (ingestionSchema.getIOConfig().isAppendToExisting() || isUseSegmentLock()) { + return new ActionBasedSegmentAllocator( + toolbox.getTaskActionClient(), + ingestionSchema.getDataSchema(), + (schema, row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> { + final Interval interval = granularitySpec + .bucketInterval(row.getTimestamp()) + .or(granularitySpec.getSegmentGranularity().bucket(row.getTimestamp())); + + final ShardSpecFactory shardSpecFactory; + if (segmentLockHelper.hasOverwritingRootGenerationPartition(interval)) { + final OverwritingRootGenerationPartitions overwritingSegmentMeta = segmentLockHelper + .getOverwritingRootGenerationPartition(interval); + if (overwritingSegmentMeta == null) { + throw new ISE("Can't find overwritingSegmentMeta for interval[%s]", interval); + } + shardSpecFactory = new NumberedOverwritingShardSpecFactory( + overwritingSegmentMeta.getStartRootPartitionId(), + overwritingSegmentMeta.getEndRootPartitionId(), + overwritingSegmentMeta.getMinorVersionForNewSegments() + ); + } else { + shardSpecFactory = NumberedShardSpecFactory.instance(); + } + + return new SurrogateAction<>( + supervisorTaskId, + new SegmentAllocateAction( + schema.getDataSource(), + row.getTimestamp(), + schema.getGranularitySpec().getQueryGranularity(), + schema.getGranularitySpec().getSegmentGranularity(), + sequenceName, + previousSegmentId, + skipSegmentLineageCheck, + shardSpecFactory, + isUseSegmentLock() ? LockGranularity.SEGMENT : LockGranularity.TIME_CHUNK + ) + ); + } + ); + } else { + return (row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> taskClient.allocateSegment( + supervisorTaskId, + row.getTimestamp() + ); + } } } @@ -273,7 +364,7 @@ private SegmentAllocator createSegmentAllocator( * * @return true if generated segments are successfully published, otherwise false */ - private List generateAndPushSegments( + private Set generateAndPushSegments( final TaskToolbox toolbox, final ParallelIndexTaskClient taskClient, final FirehoseFactory firehoseFactory, @@ -301,7 +392,7 @@ private List generateAndPushSegments( @Nullable final Long maxTotalRows = IndexTask.getValidMaxTotalRows(tuningConfig); final long pushTimeout = tuningConfig.getPushTimeout(); final boolean explicitIntervals = granularitySpec.bucketIntervals().isPresent(); - final SegmentAllocator segmentAllocator = createSegmentAllocator(toolbox, taskClient, ingestionSchema); + final SegmentAllocator segmentAllocator = createSegmentAllocator(toolbox, taskClient); try ( final Appenderator appenderator = newAppenderator(fireDepartmentMetrics, toolbox, dataSchema, tuningConfig); @@ -310,7 +401,7 @@ private List generateAndPushSegments( ) { driver.startJob(); - final List pushedSegments = new ArrayList<>(); + final Set pushedSegments = new HashSet<>(); while (firehose.hasMore()) { try { @@ -377,6 +468,15 @@ private List generateAndPushSegments( } } + private static Granularity findSegmentGranularity(GranularitySpec granularitySpec) + { + if (granularitySpec instanceof UniformGranularitySpec) { + return granularitySpec.getSegmentGranularity(); + } else { + return Granularities.ALL; + } + } + private static Appenderator newAppenderator( FireDepartmentMetrics metrics, TaskToolbox toolbox, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java index 924be846d9c9..8765884d5e8c 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java @@ -36,11 +36,11 @@ import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.actions.LockListAction; -import org.apache.druid.indexing.common.actions.LockTryAcquireAction; import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction; import org.apache.druid.indexing.common.config.TaskConfig; import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; -import org.apache.druid.indexing.common.task.AbstractTask; +import org.apache.druid.indexing.common.task.AbstractBatchIndexTask; import org.apache.druid.indexing.common.task.IndexTask; import org.apache.druid.indexing.common.task.IndexTask.IndexIngestionSpec; import org.apache.druid.indexing.common.task.IndexTask.IndexTuningConfig; @@ -50,8 +50,10 @@ import org.apache.druid.indexing.common.task.batch.parallel.ParallelIndexTaskRunner.SubTaskSpecStatus; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.segment.indexing.TuningConfig; +import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import org.apache.druid.segment.indexing.granularity.GranularitySpec; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.segment.realtime.firehose.ChatHandler; @@ -59,6 +61,7 @@ import org.apache.druid.segment.realtime.firehose.ChatHandlers; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthorizerMapper; +import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -91,7 +94,7 @@ * * @see ParallelIndexTaskRunner */ -public class ParallelIndexSupervisorTask extends AbstractTask implements ChatHandler +public class ParallelIndexSupervisorTask extends AbstractBatchIndexTask implements ChatHandler { public static final String TYPE = "index_parallel"; @@ -190,6 +193,7 @@ AuthorizerMapper getAuthorizerMapper() @VisibleForTesting ParallelIndexTaskRunner createRunner(TaskToolbox toolbox) { + this.toolbox = toolbox; if (ingestionSchema.getTuningConfig().isForceGuaranteedRollup()) { throw new UnsupportedOperationException("Perfect roll-up is not supported yet"); } else { @@ -214,26 +218,54 @@ void setRunner(ParallelIndexTaskRunner runner) @Override public boolean isReady(TaskActionClient taskActionClient) throws Exception { - final Optional> intervals = ingestionSchema.getDataSchema() - .getGranularitySpec() - .bucketIntervals(); + if (!ingestionSchema.getDataSchema().getGranularitySpec().bucketIntervals().isPresent() + && !ingestionSchema.getIOConfig().isAppendToExisting()) { + // If intervals are missing in the granularitySpec, parallel index task runs in "dynamic locking mode". + // In this mode, sub tasks ask new locks whenever they see a new row which is not covered by existing locks. + // If this task is overwriting existing segments, then we should know this task is changing segment granularity + // in advance to know what types of lock we should use. However, if intervals are missing, we can't know + // the segment granularity of existing segments until the task reads all data because we don't know what segments + // are going to be overwritten. As a result, we assume that segment granularity will be changed if intervals are + // missing force to use timeChunk lock. + addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, true); + } + return determineLockGranularityAndTryLock(taskActionClient, ingestionSchema.getDataSchema().getGranularitySpec()); + } - return !intervals.isPresent() || isReady(taskActionClient, intervals.get()); + @Override + public List findSegmentsToLock(TaskActionClient taskActionClient, List intervals) + throws IOException + { + return findInputSegments( + getDataSource(), + taskActionClient, + intervals, + ingestionSchema.getIOConfig().getFirehoseFactory() + ); } - static boolean isReady(TaskActionClient actionClient, SortedSet intervals) throws IOException + @Override + public boolean requireLockExistingSegments() { - final List locks = getTaskLocks(actionClient); - if (locks.isEmpty()) { - try { - Tasks.tryAcquireExclusiveLocks(actionClient, intervals); - } - catch (Exception e) { - log.error(e, "Failed to acquire locks for intervals[%s]", intervals); - return false; - } + return !ingestionSchema.getIOConfig().isAppendToExisting(); + } + + @Override + public boolean isPerfectRollup() + { + return false; + } + + @Nullable + @Override + public Granularity getSegmentGranularity() + { + final GranularitySpec granularitySpec = ingestionSchema.getDataSchema().getGranularitySpec(); + if (granularitySpec instanceof ArbitraryGranularitySpec) { + return null; + } else { + return granularitySpec.getSegmentGranularity(); } - return true; } @Override @@ -247,8 +279,6 @@ public void stopGracefully(TaskConfig taskConfig) @Override public TaskStatus run(TaskToolbox toolbox) throws Exception { - setToolbox(toolbox); - log.info( "Found chat handler of class[%s]", Preconditions.checkNotNull(chatHandlerProvider, "chatHandlerProvider").getClass().getName() @@ -302,9 +332,9 @@ private TaskStatus runParallel(TaskToolbox toolbox) throws Exception return TaskStatus.fromCode(getId(), Preconditions.checkNotNull(runner, "runner").run()); } - private TaskStatus runSequential(TaskToolbox toolbox) + private TaskStatus runSequential(TaskToolbox toolbox) throws Exception { - return new IndexTask( + final IndexTask indexTask = new IndexTask( getId(), getGroupId(), getTaskResource(), @@ -318,7 +348,12 @@ private TaskStatus runSequential(TaskToolbox toolbox) authorizerMapper, chatHandlerProvider, rowIngestionMetersFactory - ).run(toolbox); + ); + if (indexTask.isReady(toolbox.getTaskActionClient())) { + return indexTask.run(toolbox); + } else { + return TaskStatus.failure(getId()); + } } private static IndexTuningConfig convertToIndexTuningConfig(ParallelIndexTuningConfig tuningConfig) @@ -360,12 +395,7 @@ private static IndexTuningConfig convertToIndexTuningConfig(ParallelIndexTuningC @Consumes(SmileMediaTypes.APPLICATION_JACKSON_SMILE) public Response allocateSegment(DateTime timestamp, @Context final HttpServletRequest req) { - ChatHandlers.authorizationCheck( - req, - Action.READ, - getDataSource(), - authorizerMapper - ); + ChatHandlers.authorizationCheck(req, Action.READ, getDataSource(), authorizerMapper); if (toolbox == null) { return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity("task is not running yet").build(); @@ -383,6 +413,11 @@ public Response allocateSegment(DateTime timestamp, @Context final HttpServletRe } } + /** + * Allocate a new segment for the given timestamp locally. + * Since the segments returned by this method overwrites any existing segments, this method should be called only + * when the {@link org.apache.druid.indexing.common.LockGranularity} is {@code TIME_CHUNK}. + */ @VisibleForTesting SegmentIdWithShardSpec allocateNewSegment(DateTime timestamp) throws IOException { @@ -404,10 +439,9 @@ SegmentIdWithShardSpec allocateNewSegment(DateTime timestamp) throws IOException Interval interval; String version; - boolean justLockedInterval = false; if (bucketIntervals.isPresent()) { - // If the granularity spec has explicit intervals, we just need to find the interval (of the segment - // granularity); we already tried to lock it at task startup. + // If granularity spec has explicit intervals, we just need to find the version associated to the interval. + // This is because we should have gotten all required locks up front when the task starts up. final Optional maybeInterval = granularitySpec.bucketInterval(timestamp); if (!maybeInterval.isPresent()) { throw new IAE("Could not find interval for timestamp [%s]", timestamp); @@ -430,21 +464,17 @@ SegmentIdWithShardSpec allocateNewSegment(DateTime timestamp) throws IOException if (version == null) { // We don't have a lock for this interval, so we should lock it now. final TaskLock lock = Preconditions.checkNotNull( - toolbox.getTaskActionClient().submit(new LockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)), - "Cannot acquire a lock for interval[%s]", interval + toolbox.getTaskActionClient().submit( + new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval) + ), + "Cannot acquire a lock for interval[%s]", + interval ); version = lock.getVersion(); - justLockedInterval = true; } } final int partitionNum = Counters.getAndIncrementInt(partitionNumCountersPerInterval, interval); - if (justLockedInterval && partitionNum != 0) { - throw new ISE( - "Expected partitionNum to be 0 for interval [%s] right after locking, but got [%s]", - interval, partitionNum - ); - } return new SegmentIdWithShardSpec( dataSource, interval, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTaskClient.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTaskClient.java index 9a639d33bb57..3d2ed6b2d1e0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTaskClient.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTaskClient.java @@ -33,7 +33,7 @@ import org.joda.time.Duration; import java.io.IOException; -import java.util.List; +import java.util.Set; public class ParallelIndexTaskClient extends IndexTaskClient { @@ -84,7 +84,7 @@ public SegmentIdWithShardSpec allocateSegment(String supervisorTaskId, DateTime } } - public void report(String supervisorTaskId, List pushedSegments) + public void report(String supervisorTaskId, Set oldSegments, Set pushedSegments) { try { final FullResponseHolder response = submitSmileRequest( @@ -92,7 +92,7 @@ public void report(String supervisorTaskId, List pushedSegments) HttpMethod.POST, "report", null, - serialize(new PushedSegmentsReport(subtaskId, pushedSegments)), + serialize(new PushedSegmentsReport(subtaskId, oldSegments, pushedSegments)), true ); if (!isSuccess(response)) { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PushedSegmentsReport.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PushedSegmentsReport.java index ddb1b48c386a..e83d9b025c09 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PushedSegmentsReport.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PushedSegmentsReport.java @@ -24,7 +24,7 @@ import com.google.common.base.Preconditions; import org.apache.druid.timeline.DataSegment; -import java.util.List; +import java.util.Set; /** * This class is used in native parallel batch indexing, currently only in {@link SinglePhaseParallelIndexTaskRunner}. @@ -34,16 +34,19 @@ public class PushedSegmentsReport { private final String taskId; - private final List segments; + private final Set oldSegments; + private final Set newSegments; @JsonCreator public PushedSegmentsReport( @JsonProperty("taskId") String taskId, - @JsonProperty("segments") List segments + @JsonProperty("oldSegments") Set oldSegments, + @JsonProperty("segments") Set newSegments ) { this.taskId = Preconditions.checkNotNull(taskId, "taskId"); - this.segments = Preconditions.checkNotNull(segments, "segments"); + this.oldSegments = Preconditions.checkNotNull(oldSegments, "oldSegments"); + this.newSegments = Preconditions.checkNotNull(newSegments, "newSegments"); } @JsonProperty @@ -53,8 +56,14 @@ public String getTaskId() } @JsonProperty - public List getSegments() + public Set getOldSegments() { - return segments; + return oldSegments; + } + + @JsonProperty("segments") + public Set getNewSegments() + { + return newSegments; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseParallelIndexTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseParallelIndexTaskRunner.java index ecbd8ca78800..79648a3f6301 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseParallelIndexTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseParallelIndexTaskRunner.java @@ -47,6 +47,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -61,7 +62,7 @@ /** * An implementation of {@link ParallelIndexTaskRunner} to support best-effort roll-up. This runner can submit and * monitor multiple {@link ParallelIndexSubTask}s. - * + *

* As its name indicates, distributed indexing is done in a single phase, i.e., without shuffling intermediate data. As * a result, this task can't be used for perfect rollup. */ @@ -81,7 +82,9 @@ public class SinglePhaseParallelIndexTaskRunner implements ParallelIndexTaskRunn private final BlockingQueue> taskCompleteEvents = new LinkedBlockingDeque<>(); - /** subTaskId -> report */ + /** + * subTaskId -> report + */ private final ConcurrentHashMap segmentsMap = new ConcurrentHashMap<>(); private volatile boolean subTaskScheduleAndMonitorStopped; @@ -115,7 +118,7 @@ public TaskState run() throws Exception log.warn("There's no input split to process"); return TaskState.SUCCESS; } - + final Iterator subTaskSpecIterator = subTaskSpecIterator().iterator(); final long taskStatusCheckingPeriod = ingestionSchema.getTuningConfig().getTaskStatusCheckPeriodMs(); @@ -269,7 +272,7 @@ public void collectReport(PushedSegmentsReport report) segmentsMap.compute(report.getTaskId(), (taskId, prevReport) -> { if (prevReport != null) { Preconditions.checkState( - prevReport.getSegments().equals(report.getSegments()), + prevReport.getNewSegments().equals(report.getNewSegments()), "task[%s] sent two or more reports and previous report[%s] is different from the current one[%s]", taskId, prevReport, @@ -400,34 +403,37 @@ public TaskHistory getCompleteSubTaskSpecAttemptHistory(St private void publish(TaskToolbox toolbox) throws IOException { - final TransactionalSegmentPublisher publisher = (segments, commitMetadata) -> { - final SegmentTransactionalInsertAction action = new SegmentTransactionalInsertAction(segments); - return toolbox.getTaskActionClient().submit(action); - }; final UsedSegmentChecker usedSegmentChecker = new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()); - final Set segmentsToPublish = segmentsMap + final Set oldSegments = new HashSet<>(); + final Set newSegments = new HashSet<>(); + segmentsMap .values() - .stream() - .flatMap(report -> report.getSegments().stream()) - .collect(Collectors.toSet()); - final boolean published = segmentsToPublish.isEmpty() - || publisher.publishSegments(segmentsToPublish, null).isSuccess(); + .forEach(report -> { + oldSegments.addAll(report.getOldSegments()); + newSegments.addAll(report.getNewSegments()); + }); + final TransactionalSegmentPublisher publisher = (segmentsToBeOverwritten, segmentsToPublish, commitMetadata) -> + toolbox.getTaskActionClient().submit( + SegmentTransactionalInsertAction.overwriteAction(segmentsToBeOverwritten, segmentsToPublish) + ); + final boolean published = newSegments.isEmpty() + || publisher.publishSegments(oldSegments, newSegments, null).isSuccess(); if (published) { - log.info("Published [%d] segments", segmentsToPublish.size()); + log.info("Published [%d] segments", newSegments.size()); } else { log.info("Transaction failure while publishing segments, checking if someone else beat us to it."); final Set segmentsIdentifiers = segmentsMap .values() .stream() - .flatMap(report -> report.getSegments().stream()) + .flatMap(report -> report.getNewSegments().stream()) .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()); if (usedSegmentChecker.findUsedSegments(segmentsIdentifiers) - .equals(segmentsToPublish)) { + .equals(newSegments)) { log.info("Our segments really do exist, awaiting handoff."); } else { - throw new ISE("Failed to publish segments[%s]", segmentsToPublish); + throw new ISE("Failed to publish segments[%s]", newSegments); } } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java b/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java index 2caf91dca994..ccbcb50874bb 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java @@ -159,12 +159,14 @@ public String getDataSource() } @JsonProperty + @Nullable public Interval getInterval() { return interval; } @JsonProperty + @Nullable public List getSegments() { return segmentIds; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/CriticalAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/CriticalAction.java index 9b06a4febc8c..9a860ea681e0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/CriticalAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/CriticalAction.java @@ -20,7 +20,6 @@ package org.apache.druid.indexing.overlord; import com.google.common.base.Preconditions; -import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.task.Task; import org.joda.time.Interval; @@ -28,8 +27,7 @@ * This class represents a critical action must be done while the task's lock is guaranteed to not be revoked in the * middle of the action. * - * Implementations must not change the lock state by calling {@link TaskLockbox#lock(TaskLockType, Task, Interval)}, - * {@link TaskLockbox#lock(TaskLockType, Task, Interval, long)}, {@link TaskLockbox#tryLock(TaskLockType, Task, Interval)}, + * Implementations must not change the lock state by calling {@link TaskLockbox#lock)}, {@link TaskLockbox#tryLock)}, * or {@link TaskLockbox#unlock(Task, Interval)}. * * Also, implementations should be finished as soon as possible because all methods in {@link TaskLockbox} are blocked diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockRequest.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockRequest.java new file mode 100644 index 000000000000..987139f5be5b --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockRequest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord; + +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskLockType; +import org.joda.time.Interval; + +public interface LockRequest +{ + LockGranularity getGranularity(); + + TaskLockType getType(); + + String getGroupId(); + + String getDataSource(); + + Interval getInterval(); + + String getVersion(); + + int getPriority(); + + boolean isRevoked(); + + TaskLock toLock(); +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockRequestForNewSegment.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockRequestForNewSegment.java new file mode 100644 index 000000000000..7ce31de29086 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockRequestForNewSegment.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.timeline.partition.ShardSpecFactory; +import org.joda.time.Interval; + +import javax.annotation.Nullable; + +public class LockRequestForNewSegment implements LockRequest +{ + private final LockGranularity lockGranularity; + private final TaskLockType lockType; + private final String groupId; + private final String dataSource; + private final Interval interval; + private final ShardSpecFactory shardSpecFactory; + private final int priority; + private final String sequenceName; + @Nullable + private final String previsousSegmentId; + private final boolean skipSegmentLineageCheck; + + private String version; + + public LockRequestForNewSegment( + LockGranularity lockGranularity, + TaskLockType lockType, + String groupId, + String dataSource, + Interval interval, + ShardSpecFactory shardSpecFactory, + int priority, + String sequenceName, + @Nullable String previsousSegmentId, + boolean skipSegmentLineageCheck + ) + { + this.lockGranularity = lockGranularity; + this.lockType = lockType; + this.groupId = groupId; + this.dataSource = dataSource; + this.interval = interval; + this.shardSpecFactory = shardSpecFactory; + this.priority = priority; + this.sequenceName = sequenceName; + this.previsousSegmentId = previsousSegmentId; + this.skipSegmentLineageCheck = skipSegmentLineageCheck; + } + + @VisibleForTesting + public LockRequestForNewSegment( + LockGranularity lockGranularity, + TaskLockType lockType, + Task task, + Interval interval, + ShardSpecFactory shardSpecFactory, + String sequenceName, + @Nullable String previsousSegmentId, + boolean skipSegmentLineageCheck + ) + { + this( + lockGranularity, + lockType, + task.getGroupId(), + task.getDataSource(), + interval, + shardSpecFactory, + task.getPriority(), + sequenceName, + previsousSegmentId, + skipSegmentLineageCheck + ); + } + + @Override + public LockGranularity getGranularity() + { + return lockGranularity; + } + + @Override + public TaskLockType getType() + { + return lockType; + } + + @Override + public String getGroupId() + { + return groupId; + } + + @Override + public String getDataSource() + { + return dataSource; + } + + @Override + public Interval getInterval() + { + return interval; + } + + @Override + public int getPriority() + { + return priority; + } + + public ShardSpecFactory getShardSpecFactory() + { + return shardSpecFactory; + } + + @Override + public String getVersion() + { + if (version == null) { + version = DateTimes.nowUtc().toString(); + } + return version; + } + + @Override + public boolean isRevoked() + { + return false; + } + + @Override + public TaskLock toLock() + { + throw new UnsupportedOperationException( + "This lockRequest must be converted to SpecificSegmentLockRequest or TimeChunkLockRequest first" + + " to convert to TaskLock" + ); + } + + public String getSequenceName() + { + return sequenceName; + } + + @Nullable + public String getPrevisousSegmentId() + { + return previsousSegmentId; + } + + public boolean isSkipSegmentLineageCheck() + { + return skipSegmentLineageCheck; + } + + @Override + public String toString() + { + return "LockRequestForNewSegment{" + + "lockGranularity=" + lockGranularity + + ", lockType=" + lockType + + ", groupId='" + groupId + '\'' + + ", dataSource='" + dataSource + '\'' + + ", interval=" + interval + + ", shardSpecFactory=" + shardSpecFactory + + ", priority=" + priority + + ", sequenceName='" + sequenceName + '\'' + + ", previsousSegmentId='" + previsousSegmentId + '\'' + + ", skipSegmentLineageCheck=" + skipSegmentLineageCheck + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockResult.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockResult.java index 6372642b8fb6..4344fe324e98 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockResult.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/LockResult.java @@ -22,54 +22,63 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.indexing.common.TaskLock; -import org.apache.druid.indexing.common.TaskLockType; -import org.apache.druid.indexing.common.task.Task; -import org.joda.time.Interval; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import javax.annotation.Nullable; /** - * This class represents the result of {@link TaskLockbox#tryLock(TaskLockType, Task, Interval)}. If the lock + * This class represents the result of {@link TaskLockbox#tryLock}. If the lock * acquisition fails, the callers can tell that it was failed because it was preempted by other locks of higher * priorities or not by checking the {@link #revoked} flag. * * The {@link #revoked} flag means that consecutive lock acquisitions for the same dataSource and interval are * returning different locks because another lock of a higher priority preempted your lock at some point. In this case, * the lock acquisition must fail. - * - * @see TaskLockbox#tryLock(TaskLockType, Task, Interval) */ public class LockResult { + @Nullable private final TaskLock taskLock; private final boolean revoked; + @Nullable + private final SegmentIdWithShardSpec newSegmentId; - public static LockResult ok(TaskLock taskLock) + public static LockResult ok(TaskLock taskLock, SegmentIdWithShardSpec newSegmentId) { - return new LockResult(taskLock, false); + return new LockResult(taskLock, newSegmentId, false); } public static LockResult fail(boolean revoked) { - return new LockResult(null, revoked); + return new LockResult(null, null, revoked); } @JsonCreator public LockResult( @JsonProperty("taskLock") @Nullable TaskLock taskLock, + @JsonProperty("newSegmentId") @Nullable SegmentIdWithShardSpec newSegmentId, @JsonProperty("revoked") boolean revoked ) { this.taskLock = taskLock; + this.newSegmentId = newSegmentId; this.revoked = revoked; } @JsonProperty("taskLock") + @Nullable public TaskLock getTaskLock() { return taskLock; } + @JsonProperty("newSegmentId") + @Nullable + public SegmentIdWithShardSpec getNewSegmentId() + { + return newSegmentId; + } + @JsonProperty("revoked") public boolean isRevoked() { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SpecificSegmentLockRequest.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SpecificSegmentLockRequest.java new file mode 100644 index 000000000000..8b20f6c52fb3 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SpecificSegmentLockRequest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord; + +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.SegmentLock; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; +import org.joda.time.Interval; + +public class SpecificSegmentLockRequest implements LockRequest +{ + private final TaskLockType lockType; + private final String groupId; + private final String dataSource; + private final Interval interval; + private final int partitionId; + private final String version; + private final int priority; + private final boolean revoked; + + public SpecificSegmentLockRequest( + TaskLockType lockType, + String groupId, + String dataSource, + Interval interval, + String version, + int partitionId, + int priority, + boolean revoked + ) + { + this.lockType = lockType; + this.groupId = groupId; + this.dataSource = dataSource; + this.interval = interval; + this.version = version; + this.partitionId = partitionId; + this.priority = priority; + this.revoked = revoked; + } + + public SpecificSegmentLockRequest( + TaskLockType lockType, + Task task, + Interval interval, + String version, + int partitionId + ) + { + this(lockType, task.getGroupId(), task.getDataSource(), interval, version, partitionId, task.getPriority(), false); + } + + public SpecificSegmentLockRequest( + LockRequestForNewSegment request, + SegmentIdWithShardSpec newId + ) + { + this( + request.getType(), + request.getGroupId(), + newId.getDataSource(), + newId.getInterval(), + newId.getVersion(), + newId.getShardSpec().getPartitionNum(), + request.getPriority(), + request.isRevoked() + ); + } + + @Override + public LockGranularity getGranularity() + { + return LockGranularity.SEGMENT; + } + + @Override + public TaskLockType getType() + { + return lockType; + } + + @Override + public String getGroupId() + { + return groupId; + } + + @Override + public String getDataSource() + { + return dataSource; + } + + @Override + public Interval getInterval() + { + return interval; + } + + @Override + public String getVersion() + { + return version; + } + + @Override + public int getPriority() + { + return priority; + } + + @Override + public boolean isRevoked() + { + return revoked; + } + + public int getPartitionId() + { + return partitionId; + } + + @Override + public TaskLock toLock() + { + return new SegmentLock( + lockType, + groupId, + dataSource, + interval, + version, + partitionId, + priority, + revoked + ); + } + + @Override + public String toString() + { + return "SpecificSegmentLockRequest{" + + "lockType=" + lockType + + ", groupId='" + groupId + '\'' + + ", dataSource='" + dataSource + '\'' + + ", interval=" + interval + + ", partitionId=" + partitionId + + ", version='" + version + '\'' + + ", priority=" + priority + + ", revoked=" + revoked + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskLockbox.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskLockbox.java index 787852ddfbd5..e064ac689513 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskLockbox.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskLockbox.java @@ -29,15 +29,17 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.inject.Inject; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.SegmentLock; import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.TimeChunkLock; import org.apache.druid.indexing.common.task.Task; -import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.guava.Comparators; import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -62,9 +64,16 @@ import java.util.stream.StreamSupport; /** - * Remembers which activeTasks have locked which intervals. Tasks are permitted to lock an interval if no other task - * outside their group has locked an overlapping interval for the same datasource. When a task locks an interval, - * it is assigned a version string that it can use to publish segments. + * Remembers which activeTasks have locked which intervals or which segments. Tasks are permitted to lock an interval + * or a segment if no other task outside their group has locked an overlapping interval for the same datasource or + * the same segments. Note that TaskLockbox is also responsible for allocating segmentIds when a task requests to lock + * a new segment. Task lock might involve version assignment. + * + * - When a task locks an interval or a new segment, it is assigned a new version string that it can use to publish + * segments. + * - When a task locks a existing segment, it doesn't need to be assigned a new version. + * + * Note that tasks of higher priorities can revoke locks of tasks of lower priorities. */ public class TaskLockbox { @@ -77,6 +86,7 @@ public class TaskLockbox private final Map>>> running = new HashMap<>(); private final TaskStorage taskStorage; + private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final ReentrantLock giant = new ReentrantLock(true); private final Condition lockReleaseCondition = giant.newCondition(); @@ -88,10 +98,12 @@ public class TaskLockbox @Inject public TaskLockbox( - TaskStorage taskStorage + TaskStorage taskStorage, + IndexerMetadataStorageCoordinator metadataStorageCoordinator ) { this.taskStorage = taskStorage; + this.metadataStorageCoordinator = metadataStorageCoordinator; } /** @@ -130,8 +142,8 @@ public int compare(Pair left, Pair right) // Bookkeeping for a log message at the end int taskLockCount = 0; for (final Pair taskAndLock : byVersionOrdering.sortedCopy(storedLocks)) { - final Task task = taskAndLock.lhs; - final TaskLock savedTaskLock = taskAndLock.rhs; + final Task task = Preconditions.checkNotNull(taskAndLock.lhs, "task"); + final TaskLock savedTaskLock = Preconditions.checkNotNull(taskAndLock.rhs, "savedTaskLock"); if (savedTaskLock.getInterval().toDurationMillis() <= 0) { // "Impossible", but you never know what crazy stuff can be restored from storage. log.warn("WTF?! Got lock[%s] with empty interval for task: %s", savedTaskLock, task.getId()); @@ -141,10 +153,13 @@ public int compare(Pair left, Pair right) // Create a new taskLock if it doesn't have a proper priority, // so that every taskLock in memory has the priority. final TaskLock savedTaskLockWithPriority = savedTaskLock.getPriority() == null - ? TaskLock.withPriority(savedTaskLock, task.getPriority()) + ? savedTaskLock.withPriority(task.getPriority()) : savedTaskLock; - final TaskLockPosse taskLockPosse = createOrFindLockPosse(task, savedTaskLockWithPriority); + final TaskLockPosse taskLockPosse = verifyAndCreateOrFindLockPosse( + task, + savedTaskLockWithPriority + ); if (taskLockPosse != null) { taskLockPosse.addTask(task); @@ -188,28 +203,89 @@ public int compare(Pair left, Pair right) } } + /** + * This method is called only in {@link #syncFromStorage()} and verifies the given task and the taskLock have the same + * groupId, dataSource, and priority. + */ + private TaskLockPosse verifyAndCreateOrFindLockPosse(Task task, TaskLock taskLock) + { + giant.lock(); + + try { + Preconditions.checkArgument( + task.getGroupId().equals(taskLock.getGroupId()), + "lock groupId[%s] is different from task groupId[%s]", + taskLock.getGroupId(), + task.getGroupId() + ); + Preconditions.checkArgument( + task.getDataSource().equals(taskLock.getDataSource()), + "lock dataSource[%s] is different from task dataSource[%s]", + taskLock.getDataSource(), + task.getDataSource() + ); + final int taskPriority = task.getPriority(); + final int lockPriority = taskLock.getNonNullPriority(); + + Preconditions.checkArgument( + lockPriority == taskPriority, + "lock priority[%s] is different from task priority[%s]", + lockPriority, + taskPriority + ); + + final LockRequest request; + switch (taskLock.getGranularity()) { + case SEGMENT: + final SegmentLock segmentLock = (SegmentLock) taskLock; + request = new SpecificSegmentLockRequest( + segmentLock.getType(), + segmentLock.getGroupId(), + segmentLock.getDataSource(), + segmentLock.getInterval(), + segmentLock.getVersion(), + segmentLock.getPartitionId(), + taskPriority, + segmentLock.isRevoked() + ); + break; + case TIME_CHUNK: + final TimeChunkLock timeChunkLock = (TimeChunkLock) taskLock; + request = new TimeChunkLockRequest( + timeChunkLock.getType(), + timeChunkLock.getGroupId(), + timeChunkLock.getDataSource(), + timeChunkLock.getInterval(), + timeChunkLock.getVersion(), + taskPriority, + timeChunkLock.isRevoked() + ); + break; + default: + throw new ISE("Unknown lockGranularity[%s]", taskLock.getGranularity()); + } + + return createOrFindLockPosse(request); + } + finally { + giant.unlock(); + } + } + /** * Acquires a lock on behalf of a task. Blocks until the lock is acquired. * - * @param lockType lock type - * @param task task to acquire lock for - * @param interval interval to lock - * * @return {@link LockResult} containing a new or an existing lock if succeeded. Otherwise, {@link LockResult} with a * {@link LockResult#revoked} flag. * * @throws InterruptedException if the current thread is interrupted */ - public LockResult lock( - final TaskLockType lockType, - final Task task, - final Interval interval - ) throws InterruptedException + public LockResult lock(final Task task, final LockRequest request) throws InterruptedException { giant.lockInterruptibly(); try { LockResult lockResult; - while (!(lockResult = tryLock(lockType, task, interval)).isOk()) { + while (!(lockResult = tryLock(task, request)).isOk()) { if (lockResult.isRevoked()) { return lockResult; } @@ -225,28 +301,18 @@ public LockResult lock( /** * Acquires a lock on behalf of a task, waiting up to the specified wait time if necessary. * - * @param lockType lock type - * @param task task to acquire a lock for - * @param interval interval to lock - * @param timeoutMs maximum time to wait - * * @return {@link LockResult} containing a new or an existing lock if succeeded. Otherwise, {@link LockResult} with a * {@link LockResult#revoked} flag. * * @throws InterruptedException if the current thread is interrupted */ - public LockResult lock( - final TaskLockType lockType, - final Task task, - final Interval interval, - long timeoutMs - ) throws InterruptedException + public LockResult lock(final Task task, final LockRequest request, long timeoutMs) throws InterruptedException { long nanos = TimeUnit.MILLISECONDS.toNanos(timeoutMs); giant.lockInterruptibly(); try { LockResult lockResult; - while (!(lockResult = tryLock(lockType, task, interval)).isOk()) { + while (!(lockResult = tryLock(task, request)).isOk()) { if (nanos <= 0 || lockResult.isRevoked()) { return lockResult; } @@ -263,20 +329,12 @@ public LockResult lock( * Attempt to acquire a lock for a task, without removing it from the queue. Can safely be called multiple times on * the same task until the lock is preempted. * - * @param lockType type of lock to be acquired - * @param task task that wants a lock - * @param interval interval to lock - * * @return {@link LockResult} containing a new or an existing lock if succeeded. Otherwise, {@link LockResult} with a * {@link LockResult#revoked} flag. * * @throws IllegalStateException if the task is not a valid active task */ - public LockResult tryLock( - final TaskLockType lockType, - final Task task, - final Interval interval - ) + public LockResult tryLock(final Task task, final LockRequest request) { giant.lock(); @@ -284,18 +342,49 @@ public LockResult tryLock( if (!activeTasks.contains(task.getId())) { throw new ISE("Unable to grant lock to inactive Task [%s]", task.getId()); } - Preconditions.checkArgument(interval.toDurationMillis() > 0, "interval empty"); + Preconditions.checkArgument(request.getInterval().toDurationMillis() > 0, "interval empty"); + + SegmentIdWithShardSpec newSegmentId = null; + final LockRequest convertedRequest; + if (request instanceof LockRequestForNewSegment) { + final LockRequestForNewSegment lockRequestForNewSegment = (LockRequestForNewSegment) request; + if (lockRequestForNewSegment.getGranularity() == LockGranularity.SEGMENT) { + newSegmentId = allocateSegmentId(lockRequestForNewSegment, request.getVersion()); + if (newSegmentId == null) { + return LockResult.fail(false); + } + convertedRequest = new SpecificSegmentLockRequest(lockRequestForNewSegment, newSegmentId); + } else { + convertedRequest = new TimeChunkLockRequest(lockRequestForNewSegment); + } + } else { + convertedRequest = request; + } - final TaskLockPosse posseToUse = createOrFindLockPosse(task, interval, lockType); + final TaskLockPosse posseToUse = createOrFindLockPosse(convertedRequest); if (posseToUse != null && !posseToUse.getTaskLock().isRevoked()) { + if (request instanceof LockRequestForNewSegment) { + final LockRequestForNewSegment lockRequestForNewSegment = (LockRequestForNewSegment) request; + if (lockRequestForNewSegment.getGranularity() == LockGranularity.TIME_CHUNK) { + if (newSegmentId != null) { + throw new ISE( + "SegmentId must be allocated after getting a timeChunk lock," + + " but we already have [%s] before getting the lock?", + newSegmentId + ); + } + newSegmentId = allocateSegmentId(lockRequestForNewSegment, posseToUse.getTaskLock().getVersion()); + } + } + // Add to existing TaskLockPosse, if necessary if (posseToUse.addTask(task)) { - log.info("Added task[%s] to TaskLock[%s]", task.getId(), posseToUse.getTaskLock().getGroupId()); + log.info("Added task[%s] to TaskLock[%s]", task.getId(), posseToUse.getTaskLock()); // Update task storage facility. If it fails, revoke the lock. try { taskStorage.addLock(task.getId(), posseToUse.getTaskLock()); - return LockResult.ok(posseToUse.getTaskLock()); + return LockResult.ok(posseToUse.getTaskLock(), newSegmentId); } catch (Exception e) { log.makeAlert("Failed to persist lock in storage") @@ -304,12 +393,18 @@ public LockResult tryLock( .addData("interval", posseToUse.getTaskLock().getInterval()) .addData("version", posseToUse.getTaskLock().getVersion()) .emit(); - unlock(task, interval); + unlock( + task, + convertedRequest.getInterval(), + posseToUse.getTaskLock().getGranularity() == LockGranularity.SEGMENT + ? ((SegmentLock) posseToUse.taskLock).getPartitionId() + : null + ); return LockResult.fail(false); } } else { log.info("Task[%s] already present in TaskLock[%s]", task.getId(), posseToUse.getTaskLock().getGroupId()); - return LockResult.ok(posseToUse.getTaskLock()); + return LockResult.ok(posseToUse.getTaskLock(), newSegmentId); } } else { final boolean lockRevoked = posseToUse != null && posseToUse.getTaskLock().isRevoked(); @@ -321,226 +416,83 @@ public LockResult tryLock( } } - /** - * See {@link #createOrFindLockPosse(Task, Interval, String, TaskLockType)} - */ - @Nullable - private TaskLockPosse createOrFindLockPosse( - final Task task, - final Interval interval, - final TaskLockType lockType - ) - { - giant.lock(); - - try { - return createOrFindLockPosse(task, interval, null, lockType); - } - finally { - giant.unlock(); - } - } - - /** - * Create a new {@link TaskLockPosse} or find an existing one for the given task and interval. Note that the returned - * {@link TaskLockPosse} can hold a revoked lock. - * - * @param task task acquiring a lock - * @param interval interval to be locked - * @param preferredVersion a preferred version string - * @param lockType type of lock to be acquired - * - * @return a lock posse or null if any posse is found and a new poss cannot be created - * - * @see #createNewTaskLockPosse - */ - @Nullable - private TaskLockPosse createOrFindLockPosse( - final Task task, - final Interval interval, - @Nullable final String preferredVersion, - final TaskLockType lockType - ) + private TaskLockPosse createOrFindLockPosse(LockRequest request) { - giant.lock(); + Preconditions.checkState(!(request instanceof LockRequestForNewSegment), "Can't handle LockRequestForNewSegment"); - try { - return createOrFindLockPosse( - lockType, - task.getId(), - task.getGroupId(), - task.getDataSource(), - interval, - preferredVersion, - task.getPriority(), - false - ); - } - finally { - giant.unlock(); - } - } - - @Nullable - private TaskLockPosse createOrFindLockPosse(Task task, TaskLock taskLock) - { giant.lock(); try { - Preconditions.checkArgument( - task.getGroupId().equals(taskLock.getGroupId()), - "lock groupId[%s] is different from task groupId[%s]", - taskLock.getGroupId(), - task.getGroupId() - ); - Preconditions.checkArgument( - task.getDataSource().equals(taskLock.getDataSource()), - "lock dataSource[%s] is different from task dataSource[%s]", - taskLock.getDataSource(), - task.getDataSource() - ); - final int taskPriority = task.getPriority(); - final int lockPriority = taskLock.getNonNullPriority(); - - Preconditions.checkArgument( - lockPriority == taskPriority, - "lock priority[%s] is different from task priority[%s]", - lockPriority, - taskPriority + final List foundPosses = findLockPossesOverlapsInterval( + request.getDataSource(), + request.getInterval() ); - return createOrFindLockPosse( - taskLock.getType(), - task.getId(), - taskLock.getGroupId(), - taskLock.getDataSource(), - taskLock.getInterval(), - taskLock.getVersion(), - taskPriority, - taskLock.isRevoked() - ); - } - finally { - giant.unlock(); - } - } - - @Nullable - private TaskLockPosse createOrFindLockPosse( - TaskLockType lockType, - String taskId, - String groupId, - String dataSource, - Interval interval, - @Nullable String preferredVersion, - int priority, - boolean revoked - ) - { - giant.lock(); - - try { - final List foundPosses = findLockPossesOverlapsInterval(dataSource, interval); + final List conflictPosses = foundPosses + .stream() + .filter(taskLockPosse -> taskLockPosse.getTaskLock().conflict(request)) + .collect(Collectors.toList()); - if (foundPosses.size() > 0) { + if (conflictPosses.size() > 0) { // If we have some locks for dataSource and interval, check they can be reused. // If they can't be reused, check lock priority and revoke existing locks if possible. - final List filteredPosses = foundPosses + final List reusablePosses = foundPosses .stream() - .filter(posse -> matchGroupIdAndContainInterval(posse.taskLock, groupId, interval)) + .filter(posse -> posse.reusableFor(request)) .collect(Collectors.toList()); - if (filteredPosses.size() == 0) { + if (reusablePosses.size() == 0) { // case 1) this task doesn't have any lock, but others do - if (lockType.equals(TaskLockType.SHARED) && isAllSharedLocks(foundPosses)) { + if (request.getType().equals(TaskLockType.SHARED) && isAllSharedLocks(conflictPosses)) { // Any number of shared locks can be acquired for the same dataSource and interval. - return createNewTaskLockPosse( - lockType, - groupId, - dataSource, - interval, - preferredVersion, - priority, - revoked - ); + return createNewTaskLockPosse(request); } else { - if (isAllRevocable(foundPosses, priority)) { - // Revoke all existing locks - foundPosses.forEach(this::revokeLock); - - return createNewTaskLockPosse( - lockType, - groupId, - dataSource, - interval, - preferredVersion, - priority, - revoked - ); - } else { - final String messagePrefix; - if (preferredVersion == null) { - messagePrefix = StringUtils.format( - "Cannot create a new taskLockPosse for task[%s], interval[%s], priority[%d], revoked[%s]", - taskId, - interval, - priority, - revoked + // During a rolling update, tasks of mixed versions can be run at the same time. Old tasks would request + // timeChunkLocks while new tasks would ask segmentLocks. The below check is to allow for old and new tasks + // to get locks of different granularities if they have the same groupId. + final boolean allDifferentGranularity = conflictPosses + .stream() + .allMatch( + conflictPosse -> conflictPosse.taskLock.getGranularity() != request.getGranularity() + && conflictPosse.getTaskLock().getGroupId().equals(request.getGroupId()) + && conflictPosse.getTaskLock().getInterval().equals(request.getInterval()) ); + if (allDifferentGranularity) { + // Lock collision was because of the different granularity in the same group. + // We can add a new taskLockPosse. + return createNewTaskLockPosse(request); + } else { + if (isAllRevocable(conflictPosses, request.getPriority())) { + // Revoke all existing locks + conflictPosses.forEach(this::revokeLock); + + return createNewTaskLockPosse(request); } else { - messagePrefix = StringUtils.format( - "Cannot create a new taskLockPosse for task[%s], interval[%s]," - + " preferredVersion[%s], priority[%d], revoked[%s]", - taskId, - interval, - preferredVersion, - priority, - revoked + log.info( + "Cannot create a new taskLockPosse for request[%s] because existing locks[%s] have same or higher priorities", + request, + conflictPosses ); + return null; } - - log.info( - "%s because existing locks[%s] have same or higher priorities", - messagePrefix, - foundPosses - ); - return null; } } - } else if (filteredPosses.size() == 1) { - // case 2) we found a lock posse for the given task - final TaskLockPosse foundPosse = filteredPosses.get(0); - if (lockType.equals(foundPosse.getTaskLock().getType())) { - return foundPosse; - } else { - throw new ISE( - "Task[%s] already acquired a lock for interval[%s] but different type[%s]", - taskId, - interval, - foundPosse.getTaskLock().getType() - ); - } + } else if (reusablePosses.size() == 1) { + // case 2) we found a lock posse for the given request + return reusablePosses.get(0); } else { // case 3) we found multiple lock posses for the given task throw new ISE( "Task group[%s] has multiple locks for the same interval[%s]?", - groupId, - interval + request.getGroupId(), + request.getInterval() ); } } else { // We don't have any locks for dataSource and interval. // Let's make a new one. - return createNewTaskLockPosse( - lockType, - groupId, - dataSource, - interval, - preferredVersion, - priority, - revoked - ); + return createNewTaskLockPosse(request); } } finally { @@ -554,51 +506,18 @@ private TaskLockPosse createOrFindLockPosse( * previously assigned to the same interval. This invariant is only mostly guaranteed, however; we assume clock * monotonicity and that callers specifying {@code preferredVersion} are doing the right thing. * - * @param lockType lock type - * @param groupId group id of task - * @param dataSource data source of task - * @param interval interval to be locked - * @param preferredVersion preferred version string - * @param priority lock priority - * @param revoked indicate the lock is revoked + * @param request request to lock * * @return a new {@link TaskLockPosse} */ - private TaskLockPosse createNewTaskLockPosse( - TaskLockType lockType, - String groupId, - String dataSource, - Interval interval, - @Nullable String preferredVersion, - int priority, - boolean revoked - ) + private TaskLockPosse createNewTaskLockPosse(LockRequest request) { giant.lock(); try { - // Create new TaskLock and assign it a version. - // Assumption: We'll choose a version that is greater than any previously-chosen version for our interval. (This - // may not always be true, unfortunately. See below.) - - final String version; - - if (preferredVersion != null) { - // We have a preferred version. We'll trust our caller to not break our ordering assumptions and just use it. - version = preferredVersion; - } else { - // We are running under an interval lock right now, so just using the current time works as long as we can - // trustour clock to be monotonic and have enough resolution since the last time we created a TaskLock for - // the same interval. This may not always be true; to assure it we would need to use some method of - // timekeeping other than the wall clock. - version = DateTimes.nowUtc().toString(); - } - - final TaskLockPosse posseToUse = new TaskLockPosse( - new TaskLock(lockType, groupId, dataSource, interval, version, priority, revoked) - ); - running.computeIfAbsent(dataSource, k -> new TreeMap<>()) - .computeIfAbsent(interval.getStart(), k -> new TreeMap<>(Comparators.intervalsByStartThenEnd())) - .computeIfAbsent(interval, k -> new ArrayList<>()) + final TaskLockPosse posseToUse = new TaskLockPosse(request.toLock()); + running.computeIfAbsent(request.getDataSource(), k -> new TreeMap<>()) + .computeIfAbsent(request.getInterval().getStart(), k -> new TreeMap<>(Comparators.intervalsByStartThenEnd())) + .computeIfAbsent(request.getInterval(), k -> new ArrayList<>()) .add(posseToUse); return posseToUse; @@ -608,6 +527,19 @@ private TaskLockPosse createNewTaskLockPosse( } } + private SegmentIdWithShardSpec allocateSegmentId(LockRequestForNewSegment request, String version) + { + return metadataStorageCoordinator.allocatePendingSegment( + request.getDataSource(), + request.getSequenceName(), + request.getPrevisousSegmentId(), + request.getInterval(), + request.getShardSpecFactory(), + version, + request.isSkipSegmentLineageCheck() + ); + } + /** * Perform the given action with a guarantee that the locks of the task are not revoked in the middle of action. This * method first checks that all locks for the given task and intervals are valid and perform the right action. @@ -631,6 +563,11 @@ public T doInCriticalSection(Task task, List intervals, CriticalAc } } + /** + * Check all locks task acquired are still valid. + * It doesn't check other semantics like acquired locks are enough to overwrite existing segments. + * This kind of semantic should be checked in each caller of {@link #doInCriticalSection}. + */ private boolean isTaskLocksValid(Task task, List intervals) { giant.lock(); @@ -638,9 +575,11 @@ private boolean isTaskLocksValid(Task task, List intervals) return intervals .stream() .allMatch(interval -> { - final TaskLock lock = getOnlyTaskLockPosseContainingInterval(task, interval).getTaskLock(); + final List lockPosses = getOnlyTaskLockPosseContainingInterval(task, interval); // Tasks cannot enter the critical section with a shared lock - return !lock.isRevoked() && lock.getType() != TaskLockType.SHARED; + return lockPosses.stream().map(TaskLockPosse::getTaskLock).allMatch( + lock -> !lock.isRevoked() && lock.getType() != TaskLockType.SHARED + ); }); } finally { @@ -735,6 +674,11 @@ public TaskLock apply(TaskLockPosse taskLockPosse) } } + public void unlock(final Task task, final Interval interval) + { + unlock(task, interval, null); + } + /** * Release lock held for a task on a particular interval. Does nothing if the task does not currently * hold the mentioned lock. @@ -742,13 +686,15 @@ public TaskLock apply(TaskLockPosse taskLockPosse) * @param task task to unlock * @param interval interval to unlock */ - public void unlock(final Task task, final Interval interval) + public void unlock(final Task task, final Interval interval, @Nullable Integer partitionId) { giant.lock(); try { final String dataSource = task.getDataSource(); - final NavigableMap>> dsRunning = running.get(task.getDataSource()); + final NavigableMap>> dsRunning = running.get( + task.getDataSource() + ); if (dsRunning == null || dsRunning.isEmpty()) { return; @@ -772,48 +718,55 @@ public void unlock(final Task task, final Interval interval) for (TaskLockPosse taskLockPosse : posses) { final TaskLock taskLock = taskLockPosse.getTaskLock(); - // Remove task from live list - log.info("Removing task[%s] from TaskLock[%s]", task.getId(), taskLock.getGroupId()); - final boolean removed = taskLockPosse.removeTask(task); + final boolean match = (partitionId == null && taskLock.getGranularity() == LockGranularity.TIME_CHUNK) + || (partitionId != null + && taskLock.getGranularity() == LockGranularity.SEGMENT + && ((SegmentLock) taskLock).getPartitionId() == partitionId); - if (taskLockPosse.isTasksEmpty()) { - log.info("TaskLock is now empty: %s", taskLock); - possesHolder.remove(taskLockPosse); - } + if (match) { + // Remove task from live list + log.info("Removing task[%s] from TaskLock[%s]", task.getId(), taskLock); + final boolean removed = taskLockPosse.removeTask(task); - if (possesHolder.isEmpty()) { - intervalToPosses.remove(interval); - } + if (taskLockPosse.isTasksEmpty()) { + log.info("TaskLock is now empty: %s", taskLock); + possesHolder.remove(taskLockPosse); + } - if (intervalToPosses.isEmpty()) { - dsRunning.remove(interval.getStart()); - } + if (possesHolder.isEmpty()) { + intervalToPosses.remove(interval); + } - if (running.get(dataSource).size() == 0) { - running.remove(dataSource); - } + if (intervalToPosses.isEmpty()) { + dsRunning.remove(interval.getStart()); + } - // Wake up blocking-lock waiters - lockReleaseCondition.signalAll(); + if (running.get(dataSource).size() == 0) { + running.remove(dataSource); + } - // Remove lock from storage. If it cannot be removed, just ignore the failure. - try { - taskStorage.removeLock(task.getId(), taskLock); - } - catch (Exception e) { - log.makeAlert(e, "Failed to clean up lock from storage") - .addData("task", task.getId()) - .addData("dataSource", taskLock.getDataSource()) - .addData("interval", taskLock.getInterval()) - .addData("version", taskLock.getVersion()) - .emit(); - } + // Wake up blocking-lock waiters + lockReleaseCondition.signalAll(); - if (!removed) { - log.makeAlert("Lock release without acquire") - .addData("task", task.getId()) - .addData("interval", interval) - .emit(); + // Remove lock from storage. If it cannot be removed, just ignore the failure. + try { + taskStorage.removeLock(task.getId(), taskLock); + } + catch (Exception e) { + log.makeAlert(e, "Failed to clean up lock from storage") + .addData("task", task.getId()) + .addData("dataSource", taskLock.getDataSource()) + .addData("interval", taskLock.getInterval()) + .addData("version", taskLock.getVersion()) + .emit(); + } + + if (!removed) { + log.makeAlert("Lock release without acquire") + .addData("task", task.getId()) + .addData("interval", interval) + .emit(); + } } } } @@ -846,7 +799,13 @@ public void remove(final Task task) try { log.info("Removing task[%s] from activeTasks", task.getId()); for (final TaskLockPosse taskLockPosse : findLockPossesForTask(task)) { - unlock(task, taskLockPosse.getTaskLock().getInterval()); + unlock( + task, + taskLockPosse.getTaskLock().getInterval(), + taskLockPosse.getTaskLock().getGranularity() == LockGranularity.SEGMENT + ? ((SegmentLock) taskLockPosse.taskLock).getPartitionId() + : null + ); } } finally { @@ -939,10 +898,21 @@ private List findLockPossesOverlapsInterval(final String dataSour } @VisibleForTesting - TaskLockPosse getOnlyTaskLockPosseContainingInterval(Task task, Interval interval) + List getOnlyTaskLockPosseContainingInterval(Task task, Interval interval) { giant.lock(); + try { + return getOnlyTaskLockPosseContainingInterval(task, interval, Collections.emptySet()); + } + finally { + giant.unlock(); + } + } + @VisibleForTesting + List getOnlyTaskLockPosseContainingInterval(Task task, Interval interval, Set partitionIds) + { + giant.lock(); try { final List filteredPosses = findLockPossesContainingInterval(task.getDataSource(), interval) .stream() @@ -952,9 +922,33 @@ TaskLockPosse getOnlyTaskLockPosseContainingInterval(Task task, Interval interva if (filteredPosses.isEmpty()) { throw new ISE("Cannot find locks for task[%s] and interval[%s]", task.getId(), interval); } else if (filteredPosses.size() > 1) { - throw new ISE("There are multiple lockPosses for task[%s] and interval[%s]?", task.getId(), interval); + if (filteredPosses.stream() + .anyMatch(posse -> posse.getTaskLock().getGranularity() == LockGranularity.TIME_CHUNK)) { + throw new ISE( + "There are multiple timeChunk lockPosses for task[%s] and interval[%s]?", + task.getId(), + interval + ); + } else { + final Map partitionIdsOfLocks = new HashMap<>(); + for (TaskLockPosse posse : filteredPosses) { + final SegmentLock segmentLock = (SegmentLock) posse.getTaskLock(); + partitionIdsOfLocks.put(segmentLock.getPartitionId(), posse); + } + + if (partitionIds.stream().allMatch(partitionIdsOfLocks::containsKey)) { + return partitionIds.stream().map(partitionIdsOfLocks::get).collect(Collectors.toList()); + } else { + throw new ISE( + "Task[%s] doesn't have locks for interval[%s] partitions[%]", + task.getId(), + interval, + partitionIds.stream().filter(pid -> !partitionIdsOfLocks.containsKey(pid)).collect(Collectors.toList()) + ); + } + } } else { - return filteredPosses.get(0); + return filteredPosses; } } finally { @@ -962,10 +956,16 @@ TaskLockPosse getOnlyTaskLockPosseContainingInterval(Task task, Interval interva } } - private static boolean matchGroupIdAndContainInterval(TaskLock existingLock, String taskGroupId, Interval interval) + @VisibleForTesting + Set getActiveTasks() { - return existingLock.getInterval().contains(interval) && - existingLock.getGroupId().equals(taskGroupId); + return activeTasks; + } + + @VisibleForTesting + Map>>> getAllLocks() + { + return running; } private static boolean isAllSharedLocks(List lockPosses) @@ -985,18 +985,9 @@ private static boolean isRevocable(TaskLockPosse lockPosse, int tryLockPriority) return existingLock.isRevoked() || existingLock.getNonNullPriority() < tryLockPriority; } - @VisibleForTesting - Set getActiveTasks() - { - return activeTasks; - } - - @VisibleForTesting - Map>>> getAllLocks() - { - return running; - } - + /** + * Task locks for tasks of the same groupId + */ static class TaskLockPosse { private final TaskLock taskLock; @@ -1026,13 +1017,15 @@ TaskLock getTaskLock() boolean addTask(Task task) { - Preconditions.checkArgument( - taskLock.getGroupId().equals(task.getGroupId()), - "groupId[%s] of task[%s] is different from the existing lockPosse's groupId[%s]", - task.getGroupId(), - task.getId(), - taskLock.getGroupId() - ); + if (taskLock.getType() == TaskLockType.EXCLUSIVE) { + Preconditions.checkArgument( + taskLock.getGroupId().equals(task.getGroupId()), + "groupId[%s] of task[%s] is different from the existing lockPosse's groupId[%s]", + task.getGroupId(), + task.getId(), + taskLock.getGroupId() + ); + } Preconditions.checkArgument( taskLock.getNonNullPriority() == task.getPriority(), "priority[%s] of task[%s] is different from the existing lockPosse's priority[%s]", @@ -1060,6 +1053,36 @@ boolean isTasksEmpty() return taskIds.isEmpty(); } + boolean reusableFor(LockRequest request) + { + if (taskLock.getType() == request.getType() && taskLock.getGranularity() == request.getGranularity()) { + switch (taskLock.getType()) { + case SHARED: + // All shared lock is not reusable. Instead, a new lock posse is created for each lock request. + // See createOrFindLockPosse(). + return false; + case EXCLUSIVE: + if (request instanceof TimeChunkLockRequest) { + return taskLock.getInterval().contains(request.getInterval()) + && taskLock.getGroupId().equals(request.getGroupId()); + } else if (request instanceof SpecificSegmentLockRequest) { + final SegmentLock segmentLock = (SegmentLock) taskLock; + final SpecificSegmentLockRequest specificSegmentLockRequest = (SpecificSegmentLockRequest) request; + return segmentLock.getInterval().contains(specificSegmentLockRequest.getInterval()) + && segmentLock.getGroupId().equals(specificSegmentLockRequest.getGroupId()) + && specificSegmentLockRequest.getPartitionId() == segmentLock.getPartitionId(); + } else { + throw new ISE("Unknown request type[%s]", request); + } + //noinspection SuspiciousIndentAfterControlStatement + default: + throw new ISE("Unknown lock type[%s]", taskLock.getType()); + } + } + + return false; + } + void forEachTask(Consumer action) { Preconditions.checkNotNull(action, "action"); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TimeChunkLockRequest.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TimeChunkLockRequest.java new file mode 100644 index 000000000000..3966df66eb73 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TimeChunkLockRequest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord; + +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.TimeChunkLock; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.java.util.common.DateTimes; +import org.joda.time.Interval; + +import javax.annotation.Nullable; + +public class TimeChunkLockRequest implements LockRequest +{ + private final TaskLockType lockType; + private final String groupId; + private final String dataSource; + private final Interval interval; + @Nullable + private final String preferredVersion; + private final int priority; + private final boolean revoked; + + public TimeChunkLockRequest( + TaskLockType lockType, + Task task, + Interval interval, + @Nullable String preferredVersion + ) + { + this(lockType, task.getGroupId(), task.getDataSource(), interval, preferredVersion, task.getPriority(), false); + } + + public TimeChunkLockRequest(LockRequestForNewSegment lockRequestForNewSegment) + { + this( + lockRequestForNewSegment.getType(), + lockRequestForNewSegment.getGroupId(), + lockRequestForNewSegment.getDataSource(), + lockRequestForNewSegment.getInterval(), + lockRequestForNewSegment.getVersion(), + lockRequestForNewSegment.getPriority(), + lockRequestForNewSegment.isRevoked() + ); + } + + public TimeChunkLockRequest( + TaskLockType lockType, + String groupId, + String dataSource, + Interval interval, + @Nullable String preferredVersion, + int priority, + boolean revoked + ) + { + this.lockType = lockType; + this.groupId = groupId; + this.dataSource = dataSource; + this.interval = interval; + this.preferredVersion = preferredVersion; + this.priority = priority; + this.revoked = revoked; + } + + @Override + public LockGranularity getGranularity() + { + return LockGranularity.TIME_CHUNK; + } + + @Override + public TaskLockType getType() + { + return lockType; + } + + @Override + public String getGroupId() + { + return groupId; + } + + @Override + public String getDataSource() + { + return dataSource; + } + + @Override + public Interval getInterval() + { + return interval; + } + + @Override + public String getVersion() + { + return preferredVersion == null ? DateTimes.nowUtc().toString() : preferredVersion; + } + + @Override + public int getPriority() + { + return priority; + } + + @Override + public boolean isRevoked() + { + return revoked; + } + + @Override + public TaskLock toLock() + { + return new TimeChunkLock( + lockType, + groupId, + dataSource, + interval, + getVersion(), + priority, + revoked + ); + } + + @Override + public String toString() + { + return "TimeChunkLockRequest{" + + "lockType=" + lockType + + ", groupId='" + groupId + '\'' + + ", dataSource='" + dataSource + '\'' + + ", interval=" + interval + + ", preferredVersion='" + preferredVersion + '\'' + + ", priority=" + priority + + ", revoked=" + revoked + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTask.java index 11ed7d4d7649..0f3d2ed44469 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTask.java @@ -30,6 +30,7 @@ import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.appenderator.ActionBasedSegmentAllocator; import org.apache.druid.indexing.appenderator.ActionBasedUsedSegmentChecker; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.actions.SegmentAllocateAction; import org.apache.druid.indexing.common.actions.TaskActionClient; @@ -55,6 +56,7 @@ import org.apache.druid.segment.realtime.firehose.ChatHandler; import org.apache.druid.segment.realtime.firehose.ChatHandlerProvider; import org.apache.druid.server.security.AuthorizerMapper; +import org.apache.druid.timeline.partition.NumberedShardSpecFactory; import org.apache.druid.utils.CircularBuffer; import javax.annotation.Nullable; @@ -75,6 +77,7 @@ public abstract class SeekableStreamIndexTask savedParseExceptions; + protected final LockGranularity lockGranularityToUse; // Lazily initialized, to avoid calling it on the overlord when tasks are instantiated. // See https://github.com/apache/incubator-druid/issues/7724 for issues that can cause. @@ -114,6 +117,9 @@ public SeekableStreamIndexTask( this.authorizerMapper = authorizerMapper; this.rowIngestionMetersFactory = rowIngestionMetersFactory; this.runnerSupplier = Suppliers.memoize(this::createTaskRunner); + this.lockGranularityToUse = getContextValue(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, Tasks.DEFAULT_FORCE_TIME_CHUNK_LOCK) + ? LockGranularity.TIME_CHUNK + : LockGranularity.SEGMENT; } private static String makeTaskId(String dataSource, String type) @@ -231,7 +237,9 @@ public StreamAppenderatorDriver newDriver( schema.getGranularitySpec().getSegmentGranularity(), sequenceName, previousSegmentId, - skipSegmentLineageCheck + skipSegmentLineageCheck, + NumberedShardSpecFactory.instance(), + lockGranularityToUse ) ), toolbox.getSegmentHandoffNotifierFactory(), diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java index a9e894112d17..4dfd676cb985 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java @@ -46,11 +46,15 @@ import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.IngestionStatsAndErrorsTaskReport; import org.apache.druid.indexing.common.IngestionStatsAndErrorsTaskReportData; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskRealtimeMetricsMonitorBuilder; import org.apache.druid.indexing.common.TaskReport; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.actions.CheckPointDataSourceMetadataAction; import org.apache.druid.indexing.common.actions.ResetDataSourceMetadataAction; +import org.apache.druid.indexing.common.actions.SegmentLockAcquireAction; +import org.apache.druid.indexing.common.actions.TimeChunkLockAcquireAction; import org.apache.druid.indexing.common.stats.RowIngestionMeters; import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; import org.apache.druid.indexing.common.task.IndexTaskUtils; @@ -201,6 +205,8 @@ public enum Status private final List> publishWaitList = new ArrayList<>(); private final List> handOffWaitList = new ArrayList<>(); + private final LockGranularity lockGranularityToUse; + private volatile DateTime startTime; private volatile Status status = Status.NOT_STARTED; // this is only ever set by the task runner thread (runThread) private volatile TaskToolbox toolbox; @@ -221,7 +227,8 @@ public SeekableStreamIndexTaskRunner( final AuthorizerMapper authorizerMapper, final Optional chatHandlerProvider, final CircularBuffer savedParseExceptions, - final RowIngestionMetersFactory rowIngestionMetersFactory + final RowIngestionMetersFactory rowIngestionMetersFactory, + final LockGranularity lockGranularityToUse ) { Preconditions.checkNotNull(task); @@ -237,6 +244,7 @@ public SeekableStreamIndexTaskRunner( this.endOffsets = new ConcurrentHashMap<>(ioConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap()); this.sequences = new CopyOnWriteArrayList<>(); this.ingestionState = IngestionState.NOT_STARTED; + this.lockGranularityToUse = lockGranularityToUse; resetNextCheckpointTime(); } @@ -384,7 +392,34 @@ private TaskStatus runInternal(TaskToolbox toolbox) throws Exception driver = task.newDriver(appenderator, toolbox, fireDepartmentMetrics); // Start up, set up initial sequences. - final Object restoredMetadata = driver.startJob(); + final Object restoredMetadata = driver.startJob( + segmentId -> { + try { + if (lockGranularityToUse == LockGranularity.SEGMENT) { + return toolbox.getTaskActionClient().submit( + new SegmentLockAcquireAction( + TaskLockType.EXCLUSIVE, + segmentId.getInterval(), + segmentId.getVersion(), + segmentId.getShardSpec().getPartitionNum(), + 1000L + ) + ).isOk(); + } else { + return toolbox.getTaskActionClient().submit( + new TimeChunkLockAcquireAction( + TaskLockType.EXCLUSIVE, + segmentId.getInterval(), + 1000L + ) + ) != null; + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + ); if (restoredMetadata == null) { // no persist has happened so far // so either this is a brand new task or replacement of a failed task diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SequenceMetadata.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SequenceMetadata.java index 20eec3591ba9..17ea3c9415b4 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SequenceMetadata.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SequenceMetadata.java @@ -301,7 +301,10 @@ TransactionalSegmentPublisher createPublisher( boolean useTransaction ) { - return (segments, commitMetadata) -> { + return (mustBeNullOrEmptySegments, segmentsToPush, commitMetadata) -> { + if (mustBeNullOrEmptySegments != null && !mustBeNullOrEmptySegments.isEmpty()) { + throw new ISE("WTH? stream ingestion tasks are overwriting segments[%s]", mustBeNullOrEmptySegments); + } final Map commitMetaMap = (Map) Preconditions.checkNotNull(commitMetadata, "commitMetadata"); final SeekableStreamEndSequenceNumbers finalPartitions = runner.deserializePartitionsFromMetadata( @@ -321,8 +324,8 @@ TransactionalSegmentPublisher createPublisher( final SegmentTransactionalInsertAction action; if (useTransaction) { - action = new SegmentTransactionalInsertAction( - segments, + action = SegmentTransactionalInsertAction.appendAction( + segmentsToPush, runner.createDataSourceMetadata( new SeekableStreamStartSequenceNumbers<>( finalPartitions.getStream(), @@ -333,7 +336,7 @@ TransactionalSegmentPublisher createPublisher( runner.createDataSourceMetadata(finalPartitions) ); } else { - action = new SegmentTransactionalInsertAction(segments, null, null); + action = SegmentTransactionalInsertAction.appendAction(segmentsToPush, null, null); } return toolbox.getTaskActionClient().submit(action); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/TaskLockTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/TaskLockTest.java new file mode 100644 index 000000000000..24600488fa99 --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/TaskLockTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.Intervals; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.io.IOException; + +public class TaskLockTest +{ + private final ObjectMapper objectMapper = new DefaultObjectMapper(); + + @Test + public void testSerdeTimeChunkLock() throws IOException + { + final TimeChunkLock lock = new TimeChunkLock( + TaskLockType.EXCLUSIVE, + "groupId", + "dataSource", + Intervals.of("2019/2020"), + "version", + 100 + ); + final String json = objectMapper.writeValueAsString(lock); + final TaskLock fromJson = objectMapper.readValue(json, TaskLock.class); + + Assert.assertEquals(lock, fromJson); + } + + @Test + public void testDeserializeTimeChunkLockWithoutType() throws IOException + { + final TimeChunkLock expected = new TimeChunkLock( + TaskLockType.EXCLUSIVE, + "groupId", + "dataSource", + Intervals.of("2019/2020"), + "version", + 100 + ); + + final String json = "{\n" + + " \"type\" : \"EXCLUSIVE\",\n" + + " \"groupId\" : \"groupId\",\n" + + " \"dataSource\" : \"dataSource\",\n" + + " \"interval\" : \"2019-01-01T00:00:00.000Z/2020-01-01T00:00:00.000Z\",\n" + + " \"version\" : \"version\",\n" + + " \"priority\" : 100,\n" + + " \"revoked\" : false\n" + + "}"; + + Assert.assertEquals(expected, objectMapper.readValue(json, TaskLock.class)); + } + + @Test + public void testSerdeSegmentLock() throws IOException + { + final SegmentLock lock = new SegmentLock( + TaskLockType.EXCLUSIVE, + "groupId", + "dataSource", + Intervals.of("2019/2020"), + "version", + 0, + 100 + ); + final String json = objectMapper.writeValueAsString(lock); + final TaskLock fromJson = objectMapper.readValue(json, TaskLock.class); + + Assert.assertEquals(lock, fromJson); + } + + @Test + public void testSerdeOldLock() throws IOException + { + final OldTaskLock oldTaskLock = new OldTaskLock( + TaskLockType.EXCLUSIVE, + "groupId", + "dataSource", + Intervals.of(("2019/2020")), + "version", + 10, + true + ); + final byte[] json = objectMapper.writeValueAsBytes(oldTaskLock); + final TaskLock fromJson = objectMapper.readValue(json, TaskLock.class); + Assert.assertEquals(LockGranularity.TIME_CHUNK, fromJson.getGranularity()); + Assert.assertEquals(TaskLockType.EXCLUSIVE, fromJson.getType()); + Assert.assertEquals("groupId", fromJson.getGroupId()); + Assert.assertEquals("dataSource", fromJson.getDataSource()); + Assert.assertEquals(Intervals.of("2019/2020"), fromJson.getInterval()); + Assert.assertEquals("version", fromJson.getVersion()); + Assert.assertEquals(10, fromJson.getPriority().intValue()); + Assert.assertTrue(fromJson.isRevoked()); + } + + private static class OldTaskLock + { + private final TaskLockType type; + private final String groupId; + private final String dataSource; + private final Interval interval; + private final String version; + private final Integer priority; + private final boolean revoked; + + @JsonCreator + public OldTaskLock( + @JsonProperty("type") @Nullable TaskLockType type, + @JsonProperty("groupId") String groupId, + @JsonProperty("dataSource") String dataSource, + @JsonProperty("interval") Interval interval, + @JsonProperty("version") String version, + @JsonProperty("priority") @Nullable Integer priority, + @JsonProperty("revoked") boolean revoked + ) + { + this.type = type == null ? TaskLockType.EXCLUSIVE : type; + this.groupId = Preconditions.checkNotNull(groupId, "groupId"); + this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); + this.interval = Preconditions.checkNotNull(interval, "interval"); + this.version = Preconditions.checkNotNull(version, "version"); + this.priority = priority; + this.revoked = revoked; + } + + @JsonProperty + public TaskLockType getType() + { + return type; + } + + @JsonProperty + public String getGroupId() + { + return groupId; + } + + @JsonProperty + public String getDataSource() + { + return dataSource; + } + + @JsonProperty + public Interval getInterval() + { + return interval; + } + + @JsonProperty + public String getVersion() + { + return version; + } + + @JsonProperty + @Nullable + public Integer getPriority() + { + return priority; + } + + @JsonProperty + public boolean isRevoked() + { + return revoked; + } + + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RemoteTaskActionClientTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RemoteTaskActionClientTest.java index 10d24a9872be..e3c353f93727 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RemoteTaskActionClientTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RemoteTaskActionClientTest.java @@ -25,6 +25,7 @@ import org.apache.druid.indexing.common.RetryPolicyFactory; import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.TimeChunkLock; import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.jackson.DefaultObjectMapper; @@ -71,7 +72,7 @@ public void testSubmitSimple() throws Exception // return status code 200 and a list with size equals 1 Map responseBody = new HashMap<>(); - final List expectedLocks = Collections.singletonList(new TaskLock( + final List expectedLocks = Collections.singletonList(new TimeChunkLock( TaskLockType.SHARED, "groupId", "dataSource", @@ -91,7 +92,7 @@ public void testSubmitSimple() throws Exception EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder); EasyMock.replay(druidLeaderClient); - Task task = new NoopTask("id", null, 0, 0, null, null, null); + Task task = NoopTask.create("id", 0); RemoteTaskActionClient client = new RemoteTaskActionClient( task, druidLeaderClient, @@ -123,7 +124,7 @@ public void testSubmitWithIllegalStatusCode() throws Exception EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder); EasyMock.replay(druidLeaderClient); - Task task = new NoopTask("id", null, 0, 0, null, null, null); + Task task = NoopTask.create("id", 0); RemoteTaskActionClient client = new RemoteTaskActionClient( task, druidLeaderClient, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocateActionTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocateActionTest.java index 1debc77f6b45..5cabcf598cb6 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocateActionTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocateActionTest.java @@ -25,20 +25,27 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.SegmentLock; import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec; +import org.apache.druid.timeline.partition.HashBasedNumberedShardSpecFactory; import org.apache.druid.timeline.partition.LinearShardSpec; +import org.apache.druid.timeline.partition.LinearShardSpecFactory; import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpecFactory; +import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.apache.druid.timeline.partition.SingleDimensionShardSpec; import org.easymock.EasyMock; import org.joda.time.DateTime; @@ -47,7 +54,17 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; + +@RunWith(Parameterized.class) public class SegmentAllocateActionTest { @Rule @@ -60,6 +77,22 @@ public class SegmentAllocateActionTest private static final DateTime PARTY_TIME = DateTimes.of("1999"); private static final DateTime THE_DISTANT_FUTURE = DateTimes.of("3000"); + private final LockGranularity lockGranularity; + + @Parameterized.Parameters(name = "{0}") + public static Iterable constructorFeeder() + { + return ImmutableList.of( + new Object[]{LockGranularity.SEGMENT}, + new Object[]{LockGranularity.TIME_CHUNK} + ); + } + + public SegmentAllocateActionTest(LockGranularity lockGranularity) + { + this.lockGranularity = lockGranularity; + } + @Before public void setUp() { @@ -107,7 +140,7 @@ public void testGranularitiesFinerThanHour() @Test public void testManySegmentsSameInterval() { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getTaskLockbox().add(task); @@ -136,47 +169,87 @@ public void testManySegmentsSameInterval() id2.toString() ); - final TaskLock partyLock = Iterables.getOnlyElement( - FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) - .filter(input -> input.getInterval().contains(PARTY_TIME)) - ); + if (lockGranularity == LockGranularity.TIME_CHUNK) { + final TaskLock partyLock = Iterables.getOnlyElement( + FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) + .filter(input -> input.getInterval().contains(PARTY_TIME)) + ); - assertSameIdentifier( - id1, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(PARTY_TIME), - partyLock.getVersion(), - new NumberedShardSpec(0, 0) - ) - ); - assertSameIdentifier( - id2, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(PARTY_TIME), - partyLock.getVersion(), - new NumberedShardSpec(1, 0) - ) - ); - assertSameIdentifier( - id3, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(PARTY_TIME), - partyLock.getVersion(), - new NumberedShardSpec(2, 0) - ) - ); + assertSameIdentifier( + id1, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLock.getVersion(), + new NumberedShardSpec(0, 0) + ) + ); + assertSameIdentifier( + id2, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLock.getVersion(), + new NumberedShardSpec(1, 0) + ) + ); + assertSameIdentifier( + id3, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLock.getVersion(), + new NumberedShardSpec(2, 0) + ) + ); + } else { + final List partyTimeLocks = taskActionTestKit.getTaskLockbox() + .findLocksForTask(task) + .stream() + .filter(input -> input.getInterval().contains(PARTY_TIME)) + .collect(Collectors.toList()); + + Assert.assertEquals(3, partyTimeLocks.size()); + + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + id1.getVersion(), + new NumberedShardSpec(0, 0) + ), + id1 + ); + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + id1.getVersion(), + new NumberedShardSpec(1, 0) + ), + id2 + ); + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + id1.getVersion(), + new NumberedShardSpec(2, 0) + ), + id3 + ); + } } @Test public void testResumeSequence() { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getTaskLockbox().add(task); + final Map allocatedPartyTimeIds = new HashMap<>(); + final Map allocatedFutureIds = new HashMap<>(); final SegmentIdWithShardSpec id1 = allocate( task, PARTY_TIME, @@ -185,6 +258,8 @@ public void testResumeSequence() "s1", null ); + Assert.assertNotNull(id1); + allocatedPartyTimeIds.put(id1.getShardSpec().getPartitionNum(), id1); final SegmentIdWithShardSpec id2 = allocate( task, THE_DISTANT_FUTURE, @@ -193,6 +268,8 @@ public void testResumeSequence() "s1", id1.toString() ); + Assert.assertNotNull(id2); + allocatedFutureIds.put(id2.getShardSpec().getPartitionNum(), id2); final SegmentIdWithShardSpec id3 = allocate( task, PARTY_TIME, @@ -201,6 +278,8 @@ public void testResumeSequence() "s1", id2.toString() ); + Assert.assertNotNull(id3); + allocatedPartyTimeIds.put(id3.getShardSpec().getPartitionNum(), id3); final SegmentIdWithShardSpec id4 = allocate( task, PARTY_TIME, @@ -209,6 +288,7 @@ public void testResumeSequence() "s1", id1.toString() ); + Assert.assertNull(id4); final SegmentIdWithShardSpec id5 = allocate( task, THE_DISTANT_FUTURE, @@ -217,6 +297,8 @@ public void testResumeSequence() "s1", id1.toString() ); + Assert.assertNotNull(id5); + allocatedFutureIds.put(id5.getShardSpec().getPartitionNum(), id5); final SegmentIdWithShardSpec id6 = allocate( task, THE_DISTANT_FUTURE, @@ -225,6 +307,7 @@ public void testResumeSequence() "s1", id1.toString() ); + Assert.assertNull(id6); final SegmentIdWithShardSpec id7 = allocate( task, THE_DISTANT_FUTURE, @@ -233,71 +316,129 @@ public void testResumeSequence() "s1", id1.toString() ); + Assert.assertNotNull(id7); + allocatedFutureIds.put(id7.getShardSpec().getPartitionNum(), id7); - final TaskLock partyLock = Iterables.getOnlyElement( - FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) - .filter( - new Predicate() - { - @Override - public boolean apply(TaskLock input) + if (lockGranularity == LockGranularity.TIME_CHUNK) { + final TaskLock partyLock = Iterables.getOnlyElement( + FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) + .filter( + new Predicate() { - return input.getInterval().contains(PARTY_TIME); + @Override + public boolean apply(TaskLock input) + { + return input.getInterval().contains(PARTY_TIME); + } } - } - ) - ); - final TaskLock futureLock = Iterables.getOnlyElement( - FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) - .filter( - new Predicate() - { - @Override - public boolean apply(TaskLock input) + ) + ); + final TaskLock futureLock = Iterables.getOnlyElement( + FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) + .filter( + new Predicate() { - return input.getInterval().contains(THE_DISTANT_FUTURE); + @Override + public boolean apply(TaskLock input) + { + return input.getInterval().contains(THE_DISTANT_FUTURE); + } } - } - ) - ); + ) + ); + + assertSameIdentifier( + id1, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLock.getVersion(), + new NumberedShardSpec(0, 0) + ) + ); + assertSameIdentifier( + id2, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(THE_DISTANT_FUTURE), + futureLock.getVersion(), + new NumberedShardSpec(0, 0) + ) + ); + assertSameIdentifier( + id3, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLock.getVersion(), + new NumberedShardSpec(1, 0) + ) + ); + } else { + final List partyLocks = taskActionTestKit.getTaskLockbox() + .findLocksForTask(task) + .stream() + .filter(input -> input.getInterval().contains(PARTY_TIME)) + .collect(Collectors.toList()); + + Assert.assertEquals(2, partyLocks.size()); + final Map partitionIdToLock = new HashMap<>(); + partyLocks.forEach(lock -> { + Assert.assertEquals(LockGranularity.SEGMENT, lock.getGranularity()); + final SegmentLock segmentLock = (SegmentLock) lock; + partitionIdToLock.put(segmentLock.getPartitionId(), segmentLock); + }); + + for (Entry entry : partitionIdToLock.entrySet()) { + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + allocatedPartyTimeIds.get(entry.getKey()).getVersion(), + new NumberedShardSpec(entry.getValue().getPartitionId(), 0) + ), + allocatedPartyTimeIds.get(entry.getKey()) + ); + } + + final List futureLocks = taskActionTestKit + .getTaskLockbox() + .findLocksForTask(task) + .stream() + .filter(input -> input.getInterval().contains(THE_DISTANT_FUTURE)) + .collect(Collectors.toList()); + + Assert.assertEquals(1, futureLocks.size()); + partitionIdToLock.clear(); + futureLocks.forEach(lock -> { + Assert.assertEquals(LockGranularity.SEGMENT, lock.getGranularity()); + final SegmentLock segmentLock = (SegmentLock) lock; + partitionIdToLock.put(segmentLock.getPartitionId(), segmentLock); + }); + + for (Entry entry : partitionIdToLock.entrySet()) { + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(THE_DISTANT_FUTURE), + allocatedFutureIds.get(entry.getKey()).getVersion(), + new NumberedShardSpec(entry.getValue().getPartitionId(), 0) + ), + allocatedFutureIds.get(entry.getKey()) + ); + } + } - assertSameIdentifier( - id1, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(PARTY_TIME), - partyLock.getVersion(), - new NumberedShardSpec(0, 0) - ) - ); - assertSameIdentifier( - id2, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(THE_DISTANT_FUTURE), - futureLock.getVersion(), - new NumberedShardSpec(0, 0) - ) - ); - assertSameIdentifier( - id3, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(PARTY_TIME), - partyLock.getVersion(), - new NumberedShardSpec(1, 0) - ) - ); Assert.assertNull(id4); - assertSameIdentifier(id5, id2); + assertSameIdentifier(id2, id5); Assert.assertNull(id6); - assertSameIdentifier(id7, id2); + assertSameIdentifier(id2, id7); } @Test public void testMultipleSequences() { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getTaskLockbox().add(task); @@ -327,90 +468,161 @@ public void testMultipleSequences() "s2", id2.toString() ); - final SegmentIdWithShardSpec id6 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null); + final SegmentIdWithShardSpec id6 = allocate( + task, + PARTY_TIME, + Granularities.NONE, + Granularities.HOUR, + "s1", + null + ); - final TaskLock partyLock = Iterables.getOnlyElement( - FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) - .filter( - new Predicate() - { - @Override - public boolean apply(TaskLock input) + if (lockGranularity == LockGranularity.TIME_CHUNK) { + final TaskLock partyLock = Iterables.getOnlyElement( + FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) + .filter( + new Predicate() { - return input.getInterval().contains(PARTY_TIME); + @Override + public boolean apply(TaskLock input) + { + return input.getInterval().contains(PARTY_TIME); + } } - } - ) - ); - final TaskLock futureLock = Iterables.getOnlyElement( - FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) - .filter( - new Predicate() - { - @Override - public boolean apply(TaskLock input) + ) + ); + final TaskLock futureLock = Iterables.getOnlyElement( + FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)) + .filter( + new Predicate() { - return input.getInterval().contains(THE_DISTANT_FUTURE); + @Override + public boolean apply(TaskLock input) + { + return input.getInterval().contains(THE_DISTANT_FUTURE); + } } - } - ) - ); + ) + ); - assertSameIdentifier( - id1, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(PARTY_TIME), - partyLock.getVersion(), - new NumberedShardSpec(0, 0) - ) - ); - assertSameIdentifier( - id2, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(PARTY_TIME), - partyLock.getVersion(), - new NumberedShardSpec(1, 0) - ) - ); - assertSameIdentifier( - id3, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(PARTY_TIME), - partyLock.getVersion(), - new NumberedShardSpec(2, 0) - ) - ); - assertSameIdentifier( - id4, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(THE_DISTANT_FUTURE), - futureLock.getVersion(), - new NumberedShardSpec(0, 0) - ) - ); - assertSameIdentifier( - id5, - new SegmentIdWithShardSpec( - DATA_SOURCE, - Granularities.HOUR.bucket(THE_DISTANT_FUTURE), - futureLock.getVersion(), - new NumberedShardSpec(1, 0) - ) - ); - assertSameIdentifier( - id6, - id1 - ); + assertSameIdentifier( + id1, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLock.getVersion(), + new NumberedShardSpec(0, 0) + ) + ); + assertSameIdentifier( + id2, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLock.getVersion(), + new NumberedShardSpec(1, 0) + ) + ); + assertSameIdentifier( + id3, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLock.getVersion(), + new NumberedShardSpec(2, 0) + ) + ); + assertSameIdentifier( + id4, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(THE_DISTANT_FUTURE), + futureLock.getVersion(), + new NumberedShardSpec(0, 0) + ) + ); + assertSameIdentifier( + id5, + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(THE_DISTANT_FUTURE), + futureLock.getVersion(), + new NumberedShardSpec(1, 0) + ) + ); + } else { + final List partyLocks = taskActionTestKit.getTaskLockbox() + .findLocksForTask(task) + .stream() + .filter(input -> input.getInterval().contains(PARTY_TIME)) + .collect(Collectors.toList()); + + Assert.assertEquals(3, partyLocks.size()); + + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLocks.get(0).getVersion(), + new NumberedShardSpec(0, 0) + ), + id1 + ); + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLocks.get(1).getVersion(), + new NumberedShardSpec(1, 0) + ), + id2 + ); + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(PARTY_TIME), + partyLocks.get(2).getVersion(), + new NumberedShardSpec(2, 0) + ), + id3 + ); + + final List futureLocks = taskActionTestKit + .getTaskLockbox() + .findLocksForTask(task) + .stream() + .filter(input -> input.getInterval().contains(THE_DISTANT_FUTURE)) + .collect(Collectors.toList()); + + Assert.assertEquals(2, futureLocks.size()); + + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(THE_DISTANT_FUTURE), + futureLocks.get(0).getVersion(), + new NumberedShardSpec(0, 0) + ), + id4 + ); + assertSameIdentifier( + new SegmentIdWithShardSpec( + DATA_SOURCE, + Granularities.HOUR.bucket(THE_DISTANT_FUTURE), + futureLocks.get(1).getVersion(), + new NumberedShardSpec(1, 0) + ), + id5 + ); + } + + assertSameIdentifier(id1, id6); } @Test public void testAddToExistingLinearShardSpecsSameGranularity() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( ImmutableSet.of( @@ -437,7 +649,8 @@ public void testAddToExistingLinearShardSpecsSameGranularity() throws Exception Granularities.NONE, Granularities.HOUR, "s1", - null + null, + LinearShardSpecFactory.instance() ); final SegmentIdWithShardSpec id2 = allocate( task, @@ -445,33 +658,34 @@ public void testAddToExistingLinearShardSpecsSameGranularity() throws Exception Granularities.NONE, Granularities.HOUR, "s1", - id1.toString() + id1.toString(), + LinearShardSpecFactory.instance() ); assertSameIdentifier( - id1, new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new LinearShardSpec(2) - ) + ), + id1 ); assertSameIdentifier( - id2, new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new LinearShardSpec(3) - ) + ), + id2 ); } @Test public void testAddToExistingNumberedShardSpecsSameGranularity() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( ImmutableSet.of( @@ -510,29 +724,29 @@ public void testAddToExistingNumberedShardSpecsSameGranularity() throws Exceptio ); assertSameIdentifier( - id1, new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(2, 2) - ) + ), + id1 ); assertSameIdentifier( - id2, new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(3, 2) - ) + ), + id2 ); } @Test public void testAddToExistingNumberedShardSpecsCoarserPreferredGranularity() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( ImmutableSet.of( @@ -556,20 +770,20 @@ public void testAddToExistingNumberedShardSpecsCoarserPreferredGranularity() thr final SegmentIdWithShardSpec id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.DAY, "s1", null); assertSameIdentifier( - id1, new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(2, 2) - ) + ), + id1 ); } @Test public void testAddToExistingNumberedShardSpecsFinerPreferredGranularity() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( ImmutableSet.of( @@ -593,20 +807,20 @@ public void testAddToExistingNumberedShardSpecsFinerPreferredGranularity() throw final SegmentIdWithShardSpec id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.MINUTE, "s1", null); assertSameIdentifier( - id1, new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(2, 2) - ) + ), + id1 ); } @Test public void testCannotAddToExistingNumberedShardSpecsWithCoarserQueryGranularity() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( ImmutableSet.of( @@ -635,7 +849,7 @@ public void testCannotAddToExistingNumberedShardSpecsWithCoarserQueryGranularity @Test public void testCannotDoAnythingWithSillyQueryGranularity() { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getTaskLockbox().add(task); final SegmentIdWithShardSpec id1 = allocate(task, PARTY_TIME, Granularities.DAY, Granularities.HOUR, "s1", null); @@ -646,7 +860,7 @@ public void testCannotDoAnythingWithSillyQueryGranularity() @Test public void testCannotAddToExistingSingleDimensionShardSpecs() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( ImmutableSet.of( @@ -675,6 +889,9 @@ public void testCannotAddToExistingSingleDimensionShardSpecs() throws Exception @Test public void testSerde() throws Exception { + final ObjectMapper objectMapper = new DefaultObjectMapper(); + objectMapper.registerSubtypes(NumberedShardSpecFactory.class); + final SegmentAllocateAction action = new SegmentAllocateAction( DATA_SOURCE, PARTY_TIME, @@ -682,21 +899,71 @@ public void testSerde() throws Exception Granularities.HOUR, "s1", "prev", - false + false, + NumberedShardSpecFactory.instance(), + lockGranularity ); - final ObjectMapper objectMapper = new DefaultObjectMapper(); final SegmentAllocateAction action2 = (SegmentAllocateAction) objectMapper.readValue( objectMapper.writeValueAsBytes(action), TaskAction.class ); - Assert.assertEquals(DATA_SOURCE, action2.getDataSource()); - Assert.assertEquals(PARTY_TIME, action2.getTimestamp()); - Assert.assertEquals(Granularities.MINUTE, action2.getQueryGranularity()); - Assert.assertEquals(Granularities.HOUR, action2.getPreferredSegmentGranularity()); - Assert.assertEquals("s1", action2.getSequenceName()); - Assert.assertEquals("prev", action2.getPreviousSegmentId()); + Assert.assertEquals(action.getDataSource(), action2.getDataSource()); + Assert.assertEquals(action.getTimestamp(), action2.getTimestamp()); + Assert.assertEquals(action.getQueryGranularity(), action2.getQueryGranularity()); + Assert.assertEquals(action.getPreferredSegmentGranularity(), action2.getPreferredSegmentGranularity()); + Assert.assertEquals(action.getSequenceName(), action2.getSequenceName()); + Assert.assertEquals(action.getPreviousSegmentId(), action2.getPreviousSegmentId()); + Assert.assertEquals(action.isSkipSegmentLineageCheck(), action2.isSkipSegmentLineageCheck()); + } + + @Test + public void testWithShardSpecFactoryAndOvershadowingSegments() throws IOException + { + final Task task = NoopTask.create(); + taskActionTestKit.getTaskLockbox().add(task); + + final ObjectMapper objectMapper = new DefaultObjectMapper(); + + taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( + ImmutableSet.of( + DataSegment.builder() + .dataSource(DATA_SOURCE) + .interval(Granularities.HOUR.bucket(PARTY_TIME)) + .version(PARTY_TIME.toString()) + .shardSpec(new HashBasedNumberedShardSpec(0, 2, ImmutableList.of("dim1"), objectMapper)) + .build(), + DataSegment.builder() + .dataSource(DATA_SOURCE) + .interval(Granularities.HOUR.bucket(PARTY_TIME)) + .version(PARTY_TIME.toString()) + .shardSpec(new HashBasedNumberedShardSpec(1, 2, ImmutableList.of("dim1"), objectMapper)) + .build() + ) + ); + + final SegmentAllocateAction action = new SegmentAllocateAction( + DATA_SOURCE, + PARTY_TIME, + Granularities.MINUTE, + Granularities.HOUR, + "seq", + null, + true, + new HashBasedNumberedShardSpecFactory(ImmutableList.of("dim1"), 2), + lockGranularity + ); + final SegmentIdWithShardSpec segmentIdentifier = action.perform(task, taskActionTestKit.getTaskActionToolbox()); + Assert.assertNotNull(segmentIdentifier); + + final ShardSpec shardSpec = segmentIdentifier.getShardSpec(); + Assert.assertEquals(2, shardSpec.getPartitionNum()); + + Assert.assertTrue(shardSpec instanceof HashBasedNumberedShardSpec); + final HashBasedNumberedShardSpec hashBasedNumberedShardSpec = (HashBasedNumberedShardSpec) shardSpec; + Assert.assertEquals(2, hashBasedNumberedShardSpec.getPartitions()); + Assert.assertEquals(ImmutableList.of("dim1"), hashBasedNumberedShardSpec.getPartitionDimensions()); } private SegmentIdWithShardSpec allocate( @@ -707,6 +974,27 @@ private SegmentIdWithShardSpec allocate( final String sequenceName, final String sequencePreviousId ) + { + return allocate( + task, + timestamp, + queryGranularity, + preferredSegmentGranularity, + sequenceName, + sequencePreviousId, + NumberedShardSpecFactory.instance() + ); + } + + private SegmentIdWithShardSpec allocate( + final Task task, + final DateTime timestamp, + final Granularity queryGranularity, + final Granularity preferredSegmentGranularity, + final String sequenceName, + final String sequencePreviousId, + final ShardSpecFactory shardSpecFactory + ) { final SegmentAllocateAction action = new SegmentAllocateAction( DATA_SOURCE, @@ -715,31 +1003,28 @@ private SegmentIdWithShardSpec allocate( preferredSegmentGranularity, sequenceName, sequencePreviousId, - false + false, + shardSpecFactory, + lockGranularity ); return action.perform(task, taskActionTestKit.getTaskActionToolbox()); } - private void assertSameIdentifier(final SegmentIdWithShardSpec one, final SegmentIdWithShardSpec other) + private void assertSameIdentifier(final SegmentIdWithShardSpec expected, final SegmentIdWithShardSpec actual) { - Assert.assertEquals(one, other); - Assert.assertEquals(one.getShardSpec().getPartitionNum(), other.getShardSpec().getPartitionNum()); + Assert.assertEquals(expected, actual); + Assert.assertEquals(expected.getShardSpec().getPartitionNum(), actual.getShardSpec().getPartitionNum()); + Assert.assertEquals(expected.getShardSpec().getClass(), actual.getShardSpec().getClass()); - if (one.getShardSpec().getClass() == NumberedShardSpec.class - && other.getShardSpec().getClass() == NumberedShardSpec.class) { + if (expected.getShardSpec().getClass() == NumberedShardSpec.class + && actual.getShardSpec().getClass() == NumberedShardSpec.class) { Assert.assertEquals( - ((NumberedShardSpec) one.getShardSpec()).getPartitions(), - ((NumberedShardSpec) other.getShardSpec()).getPartitions() + ((NumberedShardSpec) expected.getShardSpec()).getPartitions(), + ((NumberedShardSpec) actual.getShardSpec()).getPartitions() ); - } else if (one.getShardSpec().getClass() == LinearShardSpec.class - && other.getShardSpec().getClass() == LinearShardSpec.class) { + } else if (expected.getShardSpec().getClass() == LinearShardSpec.class + && actual.getShardSpec().getClass() == LinearShardSpec.class) { // do nothing - } else { - throw new ISE( - "Unexpected shardSpecs [%s] and [%s]", - one.getShardSpec().getClass(), - other.getShardSpec().getClass() - ); } } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentInsertActionTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentInsertActionTest.java index abd43d986ed7..9fc905c278e5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentInsertActionTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentInsertActionTest.java @@ -26,6 +26,8 @@ import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.CriticalAction; +import org.apache.druid.indexing.overlord.LockResult; +import org.apache.druid.indexing.overlord.TimeChunkLockRequest; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; @@ -88,13 +90,19 @@ public class SegmentInsertActionTest 1024 ); + private LockResult acquireTimeChunkLock(TaskLockType lockType, Task task, Interval interval, long timeoutMs) + throws InterruptedException + { + return actionTestKit.getTaskLockbox().lock(task, new TimeChunkLockRequest(lockType, task, interval, null), timeoutMs); + } + @Test public void testSimple() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); final SegmentInsertAction action = new SegmentInsertAction(ImmutableSet.of(SEGMENT1, SEGMENT2)); actionTestKit.getTaskLockbox().add(task); - actionTestKit.getTaskLockbox().lock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); + acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); actionTestKit.getTaskLockbox().doInCriticalSection( task, Collections.singletonList(INTERVAL), @@ -121,10 +129,10 @@ public void testSimple() throws Exception @Test public void testFailBadVersion() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); final SegmentInsertAction action = new SegmentInsertAction(ImmutableSet.of(SEGMENT3)); actionTestKit.getTaskLockbox().add(task); - actionTestKit.getTaskLockbox().lock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); + acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); thrown.expect(IllegalStateException.class); thrown.expectMessage(CoreMatchers.containsString("are not covered by locks")); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertActionTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertActionTest.java index 463916f9567e..42417d680b06 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertActionTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertActionTest.java @@ -25,8 +25,10 @@ import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.LockResult; import org.apache.druid.indexing.overlord.ObjectMetadata; import org.apache.druid.indexing.overlord.SegmentPublishResult; +import org.apache.druid.indexing.overlord.TimeChunkLockRequest; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; @@ -86,14 +88,20 @@ public class SegmentTransactionalInsertActionTest 1024 ); + private LockResult acquireTimeChunkLock(TaskLockType lockType, Task task, Interval interval, long timeoutMs) + throws InterruptedException + { + return actionTestKit.getTaskLockbox().lock(task, new TimeChunkLockRequest(lockType, task, interval, null), timeoutMs); + } + @Test public void testTransactional() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); actionTestKit.getTaskLockbox().add(task); - actionTestKit.getTaskLockbox().lock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); + acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); - SegmentPublishResult result1 = new SegmentTransactionalInsertAction( + SegmentPublishResult result1 = SegmentTransactionalInsertAction.appendAction( ImmutableSet.of(SEGMENT1), new ObjectMetadata(null), new ObjectMetadata(ImmutableList.of(1)) @@ -103,7 +111,7 @@ public void testTransactional() throws Exception ); Assert.assertEquals(SegmentPublishResult.ok(ImmutableSet.of(SEGMENT1)), result1); - SegmentPublishResult result2 = new SegmentTransactionalInsertAction( + SegmentPublishResult result2 = SegmentTransactionalInsertAction.appendAction( ImmutableSet.of(SEGMENT2), new ObjectMetadata(ImmutableList.of(1)), new ObjectMetadata(ImmutableList.of(2)) @@ -130,11 +138,11 @@ public void testTransactional() throws Exception @Test public void testFailTransactional() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); + final Task task = NoopTask.create(); actionTestKit.getTaskLockbox().add(task); - actionTestKit.getTaskLockbox().lock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); + acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); - SegmentPublishResult result = new SegmentTransactionalInsertAction( + SegmentPublishResult result = SegmentTransactionalInsertAction.appendAction( ImmutableSet.of(SEGMENT1), new ObjectMetadata(ImmutableList.of(1)), new ObjectMetadata(ImmutableList.of(2)) @@ -149,10 +157,13 @@ public void testFailTransactional() throws Exception @Test public void testFailBadVersion() throws Exception { - final Task task = new NoopTask(null, null, 0, 0, null, null, null); - final SegmentTransactionalInsertAction action = new SegmentTransactionalInsertAction(ImmutableSet.of(SEGMENT3)); + final Task task = NoopTask.create(); + final SegmentTransactionalInsertAction action = SegmentTransactionalInsertAction.overwriteAction( + null, + ImmutableSet.of(SEGMENT3) + ); actionTestKit.getTaskLockbox().add(task); - actionTestKit.getTaskLockbox().lock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); + acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000); thrown.expect(IllegalStateException.class); thrown.expectMessage(CoreMatchers.containsString("are not covered by locks")); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SurrogateActionTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SurrogateActionTest.java index 31bc77392fc8..5edcbb579203 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SurrogateActionTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SurrogateActionTest.java @@ -35,8 +35,9 @@ public class SurrogateActionTest public void testSerde() throws IOException { final ObjectMapper objectMapper = new DefaultObjectMapper(); - final SurrogateAction surrogateAction = new SurrogateAction<>( - "testId", new LockTryAcquireAction(TaskLockType.EXCLUSIVE, Intervals.of("2018-01-01/2019-01-01")) + final SurrogateAction surrogateAction = new SurrogateAction<>( + "testId", + new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, Intervals.of("2018-01-01/2019-01-01")) ); final String json = objectMapper.writeValueAsString(surrogateAction); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionPreconditionsTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionPreconditionsTest.java deleted file mode 100644 index 175417cb845b..000000000000 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionPreconditionsTest.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.indexing.common.actions; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.druid.indexing.common.TaskLock; -import org.apache.druid.indexing.common.TaskLockType; -import org.apache.druid.indexing.common.config.TaskStorageConfig; -import org.apache.druid.indexing.common.task.NoopTask; -import org.apache.druid.indexing.common.task.Task; -import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage; -import org.apache.druid.indexing.overlord.TaskLockbox; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.partition.LinearShardSpec; -import org.joda.time.Interval; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -public class TaskActionPreconditionsTest -{ - private TaskLockbox lockbox; - private Task task; - private Set segments; - - @Before - public void setup() - { - lockbox = new TaskLockbox(new HeapMemoryTaskStorage(new TaskStorageConfig(null))); - task = NoopTask.create(); - lockbox.add(task); - - segments = ImmutableSet.of( - new DataSegment.Builder() - .dataSource(task.getDataSource()) - .interval(Intervals.of("2017-01-01/2017-01-02")) - .version(DateTimes.nowUtc().toString()) - .shardSpec(new LinearShardSpec(2)) - .build(), - new DataSegment.Builder() - .dataSource(task.getDataSource()) - .interval(Intervals.of("2017-01-02/2017-01-03")) - .version(DateTimes.nowUtc().toString()) - .shardSpec(new LinearShardSpec(2)) - .build(), - new DataSegment.Builder() - .dataSource(task.getDataSource()) - .interval(Intervals.of("2017-01-03/2017-01-04")) - .version(DateTimes.nowUtc().toString()) - .shardSpec(new LinearShardSpec(2)) - .build() - ); - } - - @Test - public void testCheckLockCoversSegments() - { - final List intervals = ImmutableList.of( - Intervals.of("2017-01-01/2017-01-02"), - Intervals.of("2017-01-02/2017-01-03"), - Intervals.of("2017-01-03/2017-01-04") - ); - - final Map locks = intervals.stream().collect( - Collectors.toMap( - Function.identity(), - interval -> { - final TaskLock lock = lockbox.tryLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock(); - Assert.assertNotNull(lock); - return lock; - } - ) - ); - - Assert.assertEquals(3, locks.size()); - Assert.assertTrue(TaskActionPreconditions.isLockCoversSegments(task, lockbox, segments)); - } - - @Test - public void testCheckLargeLockCoversSegments() - { - final List intervals = ImmutableList.of( - Intervals.of("2017-01-01/2017-01-04") - ); - - final Map locks = intervals.stream().collect( - Collectors.toMap( - Function.identity(), - interval -> { - final TaskLock lock = lockbox.tryLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock(); - Assert.assertNotNull(lock); - return lock; - } - ) - ); - - Assert.assertEquals(1, locks.size()); - Assert.assertTrue(TaskActionPreconditions.isLockCoversSegments(task, lockbox, segments)); - } - - @Test - public void testCheckLockCoversSegmentsWithOverlappedIntervals() - { - final List lockIntervals = ImmutableList.of( - Intervals.of("2016-12-31/2017-01-01"), - Intervals.of("2017-01-01/2017-01-02"), - Intervals.of("2017-01-02/2017-01-03") - ); - - final Map locks = lockIntervals.stream().collect( - Collectors.toMap( - Function.identity(), - interval -> { - final TaskLock lock = lockbox.tryLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock(); - Assert.assertNotNull(lock); - return lock; - } - ) - ); - - Assert.assertEquals(3, locks.size()); - Assert.assertFalse(TaskActionPreconditions.isLockCoversSegments(task, lockbox, segments)); - } -} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java index ceb65d1f31e7..e1477d603743 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java @@ -75,7 +75,6 @@ public TaskActionToolbox getTaskActionToolbox() public void before() { taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(new Period("PT24H"))); - taskLockbox = new TaskLockbox(taskStorage); testDerbyConnector = new TestDerbyConnector( Suppliers.ofInstance(new MetadataStorageConnectorConfig()), Suppliers.ofInstance(metadataStorageTablesConfig) @@ -86,6 +85,7 @@ public void before() metadataStorageTablesConfig, testDerbyConnector ); + taskLockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator); metadataSegmentManager = new SQLMetadataSegmentManager( objectMapper, Suppliers.ofInstance(new MetadataSegmentManagerConfig()), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskLocksTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskLocksTest.java new file mode 100644 index 000000000000..8b8f11ee8cf9 --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskLocksTest.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.druid.indexing.common.SegmentLock; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.TimeChunkLock; +import org.apache.druid.indexing.common.config.TaskStorageConfig; +import org.apache.druid.indexing.common.task.NoopTask; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage; +import org.apache.druid.indexing.overlord.LockResult; +import org.apache.druid.indexing.overlord.SpecificSegmentLockRequest; +import org.apache.druid.indexing.overlord.TaskLockbox; +import org.apache.druid.indexing.overlord.TimeChunkLockRequest; +import org.apache.druid.indexing.test.TestIndexerMetadataStorageCoordinator; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class TaskLocksTest +{ + private TaskLockbox lockbox; + private Task task; + + @Before + public void setup() + { + lockbox = new TaskLockbox( + new HeapMemoryTaskStorage(new TaskStorageConfig(null)), + new TestIndexerMetadataStorageCoordinator() + ); + task = NoopTask.create(); + lockbox.add(task); + } + + private Set createTimeChunkedSegments() + { + return ImmutableSet.of( + new DataSegment.Builder() + .dataSource(task.getDataSource()) + .interval(Intervals.of("2017-01-01/2017-01-02")) + .version(DateTimes.nowUtc().toString()) + .shardSpec(new LinearShardSpec(2)) + .build(), + new DataSegment.Builder() + .dataSource(task.getDataSource()) + .interval(Intervals.of("2017-01-02/2017-01-03")) + .version(DateTimes.nowUtc().toString()) + .shardSpec(new LinearShardSpec(2)) + .build(), + new DataSegment.Builder() + .dataSource(task.getDataSource()) + .interval(Intervals.of("2017-01-03/2017-01-04")) + .version(DateTimes.nowUtc().toString()) + .shardSpec(new LinearShardSpec(2)) + .build() + ); + } + + private Set createNumberedPartitionedSegments() + { + final String version = DateTimes.nowUtc().toString(); + return ImmutableSet.of( + new DataSegment.Builder() + .dataSource(task.getDataSource()) + .interval(Intervals.of("2017-01-01/2017-01-02")) + .version(version) + .shardSpec(new NumberedShardSpec(0, 0)) + .build(), + new DataSegment.Builder() + .dataSource(task.getDataSource()) + .interval(Intervals.of("2017-01-01/2017-01-02")) + .version(version) + .shardSpec(new NumberedShardSpec(1, 0)) + .build(), + new DataSegment.Builder() + .dataSource(task.getDataSource()) + .interval(Intervals.of("2017-01-01/2017-01-02")) + .version(version) + .shardSpec(new NumberedShardSpec(2, 0)) + .build(), + new DataSegment.Builder() + .dataSource(task.getDataSource()) + .interval(Intervals.of("2017-01-01/2017-01-02")) + .version(version) + .shardSpec(new NumberedShardSpec(3, 0)) + .build(), + new DataSegment.Builder() + .dataSource(task.getDataSource()) + .interval(Intervals.of("2017-01-01/2017-01-02")) + .version(version) + .shardSpec(new NumberedShardSpec(4, 0)) + .build() + ); + } + + private LockResult tryTimeChunkLock(Task task, Interval interval) + { + return lockbox.tryLock(task, new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task, interval, null)); + } + + private LockResult trySegmentLock(Task task, Interval interval, String version, int partitonId) + { + return lockbox.tryLock( + task, + new SpecificSegmentLockRequest(TaskLockType.EXCLUSIVE, task, interval, version, partitonId) + ); + } + + @Test + public void testCheckLockCoversSegments() + { + final Set segments = createTimeChunkedSegments(); + final List intervals = ImmutableList.of( + Intervals.of("2017-01-01/2017-01-02"), + Intervals.of("2017-01-02/2017-01-03"), + Intervals.of("2017-01-03/2017-01-04") + ); + + final Map locks = intervals.stream().collect( + Collectors.toMap( + Function.identity(), + interval -> { + final TaskLock lock = tryTimeChunkLock(task, interval).getTaskLock(); + Assert.assertNotNull(lock); + return lock; + } + ) + ); + + Assert.assertEquals(3, locks.size()); + Assert.assertTrue(TaskLocks.isLockCoversSegments(task, lockbox, segments)); + } + + @Test + public void testCheckSegmentLockCoversSegments() + { + final Set segments = createNumberedPartitionedSegments(); + final Interval interval = Intervals.of("2017-01-01/2017-01-02"); + + final String version = DateTimes.nowUtc().toString(); + final List locks = IntStream + .range(0, 5) + .mapToObj( + partitionId -> { + final TaskLock lock = trySegmentLock(task, interval, version, partitionId).getTaskLock(); + Assert.assertNotNull(lock); + return lock; + } + ).collect(Collectors.toList()); + + Assert.assertEquals(5, locks.size()); + Assert.assertTrue(TaskLocks.isLockCoversSegments(task, lockbox, segments)); + } + + @Test + public void testCheckLargeLockCoversSegments() + { + final Set segments = createTimeChunkedSegments(); + final List intervals = ImmutableList.of( + Intervals.of("2017-01-01/2017-01-04") + ); + + final Map locks = intervals.stream().collect( + Collectors.toMap( + Function.identity(), + interval -> { + final TaskLock lock = tryTimeChunkLock(task, interval).getTaskLock(); + Assert.assertNotNull(lock); + return lock; + } + ) + ); + + Assert.assertEquals(1, locks.size()); + Assert.assertTrue(TaskLocks.isLockCoversSegments(task, lockbox, segments)); + } + + @Test + public void testCheckLockCoversSegmentsWithOverlappedIntervals() + { + final Set segments = createTimeChunkedSegments(); + final List lockIntervals = ImmutableList.of( + Intervals.of("2016-12-31/2017-01-01"), + Intervals.of("2017-01-01/2017-01-02"), + Intervals.of("2017-01-02/2017-01-03") + ); + + final Map locks = lockIntervals.stream().collect( + Collectors.toMap( + Function.identity(), + interval -> { + final TaskLock lock = tryTimeChunkLock(task, interval).getTaskLock(); + Assert.assertNotNull(lock); + return lock; + } + ) + ); + + Assert.assertEquals(3, locks.size()); + Assert.assertFalse(TaskLocks.isLockCoversSegments(task, lockbox, segments)); + } + + @Test + public void testFindLocksForSegments() + { + final Set segments = createTimeChunkedSegments(); + final List intervals = ImmutableList.of( + Intervals.of("2017-01-01/2017-01-02"), + Intervals.of("2017-01-02/2017-01-03"), + Intervals.of("2017-01-03/2017-01-04") + ); + + final Map locks = intervals.stream().collect( + Collectors.toMap( + Function.identity(), + interval -> { + final TaskLock lock = tryTimeChunkLock(task, interval).getTaskLock(); + Assert.assertNotNull(lock); + return lock; + } + ) + ); + + Assert.assertEquals(3, locks.size()); + Assert.assertEquals( + ImmutableList.of( + newTimeChunkLock(intervals.get(0), locks.get(intervals.get(0)).getVersion()), + newTimeChunkLock(intervals.get(1), locks.get(intervals.get(1)).getVersion()), + newTimeChunkLock(intervals.get(2), locks.get(intervals.get(2)).getVersion()) + ), + TaskLocks.findLocksForSegments(task, lockbox, segments) + ); + } + + @Test + public void testFindSegmentLocksForSegments() + { + final Set segments = createNumberedPartitionedSegments(); + final Interval interval = Intervals.of("2017-01-01/2017-01-02"); + + final String version = DateTimes.nowUtc().toString(); + final List locks = IntStream + .range(0, 5) + .mapToObj( + partitionId -> { + final TaskLock lock = trySegmentLock(task, interval, version, partitionId).getTaskLock(); + Assert.assertNotNull(lock); + return lock; + } + ).collect(Collectors.toList()); + + Assert.assertEquals(5, locks.size()); + Assert.assertEquals( + ImmutableList.of( + newSegmentLock(interval, locks.get(0).getVersion(), 0), + newSegmentLock(interval, locks.get(0).getVersion(), 1), + newSegmentLock(interval, locks.get(0).getVersion(), 2), + newSegmentLock(interval, locks.get(0).getVersion(), 3), + newSegmentLock(interval, locks.get(0).getVersion(), 4) + ), + TaskLocks.findLocksForSegments(task, lockbox, segments) + ); + } + + private TimeChunkLock newTimeChunkLock(Interval interval, String version) + { + return new TimeChunkLock( + TaskLockType.EXCLUSIVE, + task.getGroupId(), + task.getDataSource(), + interval, + version, + task.getPriority() + ); + } + + private SegmentLock newSegmentLock(Interval interval, String version, int partitionId) + { + return new SegmentLock( + TaskLockType.EXCLUSIVE, + task.getGroupId(), + task.getDataSource(), + interval, + version, + partitionId, + task.getPriority() + ); + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/LockAcquireActionTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TimeChunkLockAcquireActionTest.java similarity index 84% rename from indexing-service/src/test/java/org/apache/druid/indexing/common/actions/LockAcquireActionTest.java rename to indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TimeChunkLockAcquireActionTest.java index 4f64f384840f..7caaa6361dfc 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/LockAcquireActionTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TimeChunkLockAcquireActionTest.java @@ -32,7 +32,7 @@ import java.io.IOException; -public class LockAcquireActionTest +public class TimeChunkLockAcquireActionTest { @Rule public TaskActionTestKit actionTestKit = new TaskActionTestKit(); @@ -42,14 +42,14 @@ public class LockAcquireActionTest @Test public void testSerdeWithAllFields() throws IOException { - final LockAcquireAction expected = new LockAcquireAction( + final TimeChunkLockAcquireAction expected = new TimeChunkLockAcquireAction( TaskLockType.SHARED, Intervals.of("2017-01-01/2017-01-02"), 1000 ); final byte[] bytes = mapper.writeValueAsBytes(expected); - final LockAcquireAction actual = mapper.readValue(bytes, LockAcquireAction.class); + final TimeChunkLockAcquireAction actual = mapper.readValue(bytes, TimeChunkLockAcquireAction.class); Assert.assertEquals(expected.getType(), actual.getType()); Assert.assertEquals(expected.getInterval(), actual.getInterval()); Assert.assertEquals(expected.getTimeoutMs(), actual.getTimeoutMs()); @@ -60,8 +60,8 @@ public void testSerdeFromJsonWithMissingFields() throws IOException { final String json = "{ \"type\": \"lockAcquire\", \"interval\" : \"2017-01-01/2017-01-02\" }"; - final LockAcquireAction actual = mapper.readValue(json, LockAcquireAction.class); - final LockAcquireAction expected = new LockAcquireAction( + final TimeChunkLockAcquireAction actual = mapper.readValue(json, TimeChunkLockAcquireAction.class); + final TimeChunkLockAcquireAction expected = new TimeChunkLockAcquireAction( TaskLockType.EXCLUSIVE, Intervals.of("2017-01-01/2017-01-02"), 0 @@ -75,7 +75,7 @@ public void testSerdeFromJsonWithMissingFields() throws IOException public void testWithLockType() { final Task task = NoopTask.create(); - final LockAcquireAction action = new LockAcquireAction( + final TimeChunkLockAcquireAction action = new TimeChunkLockAcquireAction( TaskLockType.EXCLUSIVE, Intervals.of("2017-01-01/2017-01-02"), 1000 @@ -90,7 +90,7 @@ public void testWithLockType() public void testWithoutLockType() { final Task task = NoopTask.create(); - final LockAcquireAction action = new LockAcquireAction( + final TimeChunkLockAcquireAction action = new TimeChunkLockAcquireAction( null, Intervals.of("2017-01-01/2017-01-02"), 1000 diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/LockTryAcquireActionTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TimeChunkLockTryAcquireActionTest.java similarity index 82% rename from indexing-service/src/test/java/org/apache/druid/indexing/common/actions/LockTryAcquireActionTest.java rename to indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TimeChunkLockTryAcquireActionTest.java index 9b1077a9346a..c6b65da1e0ea 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/LockTryAcquireActionTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TimeChunkLockTryAcquireActionTest.java @@ -32,7 +32,7 @@ import java.io.IOException; -public class LockTryAcquireActionTest +public class TimeChunkLockTryAcquireActionTest { @Rule public TaskActionTestKit actionTestKit = new TaskActionTestKit(); @@ -42,13 +42,13 @@ public class LockTryAcquireActionTest @Test public void testSerdeWithAllFields() throws IOException { - final LockTryAcquireAction expected = new LockTryAcquireAction( + final TimeChunkLockTryAcquireAction expected = new TimeChunkLockTryAcquireAction( TaskLockType.SHARED, Intervals.of("2017-01-01/2017-01-02") ); final byte[] bytes = mapper.writeValueAsBytes(expected); - final LockTryAcquireAction actual = mapper.readValue(bytes, LockTryAcquireAction.class); + final TimeChunkLockTryAcquireAction actual = mapper.readValue(bytes, TimeChunkLockTryAcquireAction.class); Assert.assertEquals(expected.getType(), actual.getType()); Assert.assertEquals(expected.getInterval(), actual.getInterval()); } @@ -58,8 +58,8 @@ public void testSerdeFromJsonWithMissingFields() throws IOException { final String json = "{ \"type\": \"lockTryAcquire\", \"interval\" : \"2017-01-01/2017-01-02\" }"; - final LockTryAcquireAction actual = mapper.readValue(json, LockTryAcquireAction.class); - final LockTryAcquireAction expected = new LockTryAcquireAction( + final TimeChunkLockTryAcquireAction actual = mapper.readValue(json, TimeChunkLockTryAcquireAction.class); + final TimeChunkLockTryAcquireAction expected = new TimeChunkLockTryAcquireAction( TaskLockType.EXCLUSIVE, Intervals.of("2017-01-01/2017-01-02") ); @@ -71,7 +71,7 @@ public void testSerdeFromJsonWithMissingFields() throws IOException public void testWithLockType() { final Task task = NoopTask.create(); - final LockTryAcquireAction action = new LockTryAcquireAction( + final TimeChunkLockTryAcquireAction action = new TimeChunkLockTryAcquireAction( TaskLockType.EXCLUSIVE, Intervals.of("2017-01-01/2017-01-02") ); @@ -85,7 +85,7 @@ public void testWithLockType() public void testWithoutLockType() { final Task task = NoopTask.create(); - final LockTryAcquireAction action = new LockTryAcquireAction( + final TimeChunkLockTryAcquireAction action = new TimeChunkLockTryAcquireAction( null, Intervals.of("2017-01-01/2017-01-02") ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java index 7628dea75362..72e16a55b60a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java @@ -1375,7 +1375,6 @@ private AppenderatorDriverRealtimeIndexTask makeRealtimeTask( final Long maxTotalRows ) { - ObjectMapper objectMapper = new DefaultObjectMapper(); DataSchema dataSchema = new DataSchema( "test_ds", TestHelper.makeJsonMapper().convertValue( @@ -1470,8 +1469,6 @@ private void awaitHandoffs() throws InterruptedException private void makeToolboxFactory(final File directory) { taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null)); - taskLockbox = new TaskLockbox(taskStorage); - publishedSegments = new CopyOnWriteArrayList<>(); ObjectMapper mapper = new DefaultObjectMapper(); @@ -1519,6 +1516,8 @@ public SegmentPublishResult announceHistoricalSegments( return result; } }; + + taskLockbox = new TaskLockbox(taskStorage, mdc); final TaskConfig taskConfig = new TaskConfig(directory.getPath(), null, null, 50000, null, true, null, null, null); final TaskActionToolbox taskActionToolbox = new TaskActionToolbox( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java index 9e5ab4c1216d..894ee0835da1 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java @@ -28,13 +28,14 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.ParseSpec; import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.RetryPolicyConfig; import org.apache.druid.indexing.common.RetryPolicyFactory; import org.apache.druid.indexing.common.SegmentLoaderFactory; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.TestUtils; -import org.apache.druid.indexing.common.actions.LocalTaskActionClient; import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; import org.apache.druid.indexing.common.task.CompactionTask.Builder; import org.apache.druid.java.util.common.ISE; @@ -43,7 +44,6 @@ import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; -import org.apache.druid.segment.loading.DataSegmentPusher; import org.apache.druid.segment.loading.LocalDataSegmentPuller; import org.apache.druid.segment.loading.LocalDataSegmentPusher; import org.apache.druid.segment.loading.LocalDataSegmentPusherConfig; @@ -55,7 +55,9 @@ import org.apache.druid.segment.loading.StorageLocationConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.NumberedOverwriteShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.apache.druid.timeline.partition.PartitionIds; import org.joda.time.Interval; import org.junit.After; import org.junit.Assert; @@ -64,19 +66,28 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import javax.annotation.Nullable; import java.io.BufferedWriter; import java.io.File; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +@RunWith(Parameterized.class) public class CompactionTaskRunTest extends IngestionTestBase { + public static final String DATA_SOURCE = "test"; + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -100,13 +111,23 @@ public class CompactionTaskRunTest extends IngestionTestBase 0 ); - private RowIngestionMetersFactory rowIngestionMetersFactory; - private CoordinatorClient coordinatorClient; - private SegmentLoaderFactory segmentLoaderFactory; + @Parameterized.Parameters(name = "{0}") + public static Iterable constructorFeeder() + { + return ImmutableList.of( + new Object[]{LockGranularity.TIME_CHUNK}, + new Object[]{LockGranularity.SEGMENT} + ); + } + + private static final RetryPolicyFactory retryPolicyFactory = new RetryPolicyFactory(new RetryPolicyConfig()); + private final RowIngestionMetersFactory rowIngestionMetersFactory; + private final CoordinatorClient coordinatorClient; + private final SegmentLoaderFactory segmentLoaderFactory; + private final LockGranularity lockGranularity; private ExecutorService exec; - private static RetryPolicyFactory retryPolicyFactory = new RetryPolicyFactory(new RetryPolicyConfig()); - public CompactionTaskRunTest() + public CompactionTaskRunTest(LockGranularity lockGranularity) { TestUtils testUtils = new TestUtils(); rowIngestionMetersFactory = testUtils.getRowIngestionMetersFactory(); @@ -119,6 +140,7 @@ public List getDatabaseSegmentDataSourceSegments(String dataSource, } }; segmentLoaderFactory = new SegmentLoaderFactory(getIndexIO(), getObjectMapper()); + this.lockGranularity = lockGranularity; } @Before @@ -161,8 +183,18 @@ public void testRun() throws Exception Assert.assertEquals(3, segments.size()); for (int i = 0; i < 3; i++) { - Assert.assertEquals(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), segments.get(i).getInterval()); - Assert.assertEquals(new NumberedShardSpec(0, 0), segments.get(i).getShardSpec()); + Assert.assertEquals( + Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), + segments.get(i).getInterval() + ); + if (lockGranularity == LockGranularity.SEGMENT) { + Assert.assertEquals( + new NumberedOverwriteShardSpec(32768, 0, 2, (short) 1, (short) 1), + segments.get(i).getShardSpec() + ); + } else { + Assert.assertEquals(new NumberedShardSpec(0, 0), segments.get(i).getShardSpec()); + } } } @@ -194,8 +226,18 @@ public void testRunCompactionTwice() throws Exception Assert.assertEquals(3, segments.size()); for (int i = 0; i < 3; i++) { - Assert.assertEquals(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), segments.get(i).getInterval()); - Assert.assertEquals(new NumberedShardSpec(0, 0), segments.get(i).getShardSpec()); + Assert.assertEquals( + Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), + segments.get(i).getInterval() + ); + if (lockGranularity == LockGranularity.SEGMENT) { + Assert.assertEquals( + new NumberedOverwriteShardSpec(PartitionIds.NON_ROOT_GEN_START_PARTITION_ID, 0, 2, (short) 1, (short) 1), + segments.get(i).getShardSpec() + ); + } else { + Assert.assertEquals(new NumberedShardSpec(0, 0), segments.get(i).getShardSpec()); + } } final CompactionTask compactionTask2 = builder @@ -210,8 +252,119 @@ public void testRunCompactionTwice() throws Exception Assert.assertEquals(3, segments.size()); for (int i = 0; i < 3; i++) { - Assert.assertEquals(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), segments.get(i).getInterval()); - Assert.assertEquals(new NumberedShardSpec(0, 0), segments.get(i).getShardSpec()); + Assert.assertEquals( + Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), + segments.get(i).getInterval() + ); + if (lockGranularity == LockGranularity.SEGMENT) { + Assert.assertEquals( + new NumberedOverwriteShardSpec( + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID + 1, + 0, + 2, + (short) 2, + (short) 1 + ), + segments.get(i).getShardSpec() + ); + } else { + Assert.assertEquals(new NumberedShardSpec(0, 0), segments.get(i).getShardSpec()); + } + } + } + + @Test + public void testRunIndexAndCompactAtTheSameTimeForDifferentInterval() throws Exception + { + runIndexTask(); + + final Builder builder = new Builder( + DATA_SOURCE, + getObjectMapper(), + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + null, + rowIngestionMetersFactory, + coordinatorClient, + segmentLoaderFactory, + retryPolicyFactory + ); + + final CompactionTask compactionTask = builder + .interval(Intervals.of("2014-01-01T00:00:00/2014-01-02T03:00:00")) + .build(); + + File tmpDir = temporaryFolder.newFolder(); + File tmpFile = File.createTempFile("druid", "index", tmpDir); + + try (BufferedWriter writer = Files.newWriter(tmpFile, StandardCharsets.UTF_8)) { + writer.write("2014-01-01T03:00:10Z,a,1\n"); + writer.write("2014-01-01T03:00:10Z,b,2\n"); + writer.write("2014-01-01T03:00:10Z,c,3\n"); + writer.write("2014-01-01T04:00:20Z,a,1\n"); + writer.write("2014-01-01T04:00:20Z,b,2\n"); + writer.write("2014-01-01T04:00:20Z,c,3\n"); + writer.write("2014-01-01T05:00:30Z,a,1\n"); + writer.write("2014-01-01T05:00:30Z,b,2\n"); + writer.write("2014-01-01T05:00:30Z,c,3\n"); + } + + IndexTask indexTask = new IndexTask( + null, + null, + IndexTaskTest.createIngestionSpec( + getObjectMapper(), + tmpDir, + DEFAULT_PARSE_SPEC, + new UniformGranularitySpec( + Granularities.HOUR, + Granularities.MINUTE, + null + ), + IndexTaskTest.createTuningConfig(2, 2, null, 2L, null, null, false, true), + false + ), + null, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + null, + rowIngestionMetersFactory + ); + + final Future>> compactionFuture = exec.submit( + () -> runTask(compactionTask) + ); + + final Future>> indexFuture = exec.submit( + () -> runTask(indexTask) + ); + + Assert.assertTrue(indexFuture.get().lhs.isSuccess()); + + List segments = indexFuture.get().rhs; + Assert.assertEquals(6, segments.size()); + + for (int i = 0; i < 6; i++) { + Assert.assertEquals(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", 3 + i / 2, 3 + i / 2 + 1), segments.get(i).getInterval()); + Assert.assertEquals(new NumberedShardSpec(i % 2, 0), segments.get(i).getShardSpec()); + } + + Assert.assertTrue(compactionFuture.get().lhs.isSuccess()); + + segments = compactionFuture.get().rhs; + Assert.assertEquals(3, segments.size()); + + for (int i = 0; i < 3; i++) { + Assert.assertEquals( + Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), + segments.get(i).getInterval() + ); + if (lockGranularity == LockGranularity.SEGMENT) { + Assert.assertEquals( + new NumberedOverwriteShardSpec(PartitionIds.NON_ROOT_GEN_START_PARTITION_ID, 0, 2, (short) 1, (short) 1), + segments.get(i).getShardSpec() + ); + } else { + Assert.assertEquals(new NumberedShardSpec(0, 0), segments.get(i).getShardSpec()); + } } } @@ -267,7 +420,195 @@ public void testWithSegmentGranularity() throws Exception } } + @Test + public void testCompactThenAppend() throws Exception + { + runIndexTask(); + + final Builder builder = new Builder( + DATA_SOURCE, + getObjectMapper(), + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + null, + rowIngestionMetersFactory, + coordinatorClient, + segmentLoaderFactory, + retryPolicyFactory + ); + + final CompactionTask compactionTask = builder + .interval(Intervals.of("2014-01-01/2014-01-02")) + .build(); + + final Set expectedSegments = new HashSet<>(); + final Pair> compactionResult = runTask(compactionTask); + Assert.assertTrue(compactionResult.lhs.isSuccess()); + expectedSegments.addAll(compactionResult.rhs); + + final Pair> appendResult = runAppendTask(); + Assert.assertTrue(appendResult.lhs.isSuccess()); + expectedSegments.addAll(appendResult.rhs); + + final Set usedSegments = new HashSet<>( + getStorageCoordinator().getUsedSegmentsForIntervals( + DATA_SOURCE, + Collections.singletonList(Intervals.of("2014-01-01/2014-01-02")) + ) + ); + + Assert.assertEquals(expectedSegments, usedSegments); + } + + @Test + public void testRunIndexAndCompactForSameSegmentAtTheSameTime() throws Exception + { + runIndexTask(); + + // make sure that indexTask becomes ready first, then compactionTask becomes ready, then indexTask runs + final CountDownLatch compactionTaskReadyLatch = new CountDownLatch(1); + final CountDownLatch indexTaskStartLatch = new CountDownLatch(1); + final Future>> indexFuture = exec.submit( + () -> runIndexTask(compactionTaskReadyLatch, indexTaskStartLatch, false) + ); + + final Builder builder = new Builder( + DATA_SOURCE, + getObjectMapper(), + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + null, + rowIngestionMetersFactory, + coordinatorClient, + segmentLoaderFactory, + retryPolicyFactory + ); + + final CompactionTask compactionTask = builder + .interval(Intervals.of("2014-01-01T00:00:00/2014-01-02T03:00:00")) + .build(); + + final Future>> compactionFuture = exec.submit( + () -> { + compactionTaskReadyLatch.await(); + return runTask(compactionTask, indexTaskStartLatch, null); + } + ); + + Assert.assertTrue(indexFuture.get().lhs.isSuccess()); + + List segments = indexFuture.get().rhs; + Assert.assertEquals(6, segments.size()); + + for (int i = 0; i < 6; i++) { + Assert.assertEquals( + Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i / 2, i / 2 + 1), + segments.get(i).getInterval() + ); + if (lockGranularity == LockGranularity.SEGMENT) { + Assert.assertEquals( + new NumberedOverwriteShardSpec( + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID + i % 2, + 0, + 2, + (short) 1, + (short) 2 + ), + segments.get(i).getShardSpec() + ); + } else { + Assert.assertEquals(new NumberedShardSpec(i % 2, 0), segments.get(i).getShardSpec()); + } + } + + final Pair> compactionResult = compactionFuture.get(); + Assert.assertEquals(TaskState.FAILED, compactionResult.lhs.getStatusCode()); + } + + @Test + public void testRunIndexAndCompactForSameSegmentAtTheSameTime2() throws Exception + { + runIndexTask(); + + final Builder builder = new Builder( + DATA_SOURCE, + getObjectMapper(), + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + null, + rowIngestionMetersFactory, + coordinatorClient, + segmentLoaderFactory, + retryPolicyFactory + ); + + final CompactionTask compactionTask = builder + .interval(Intervals.of("2014-01-01T00:00:00/2014-01-02T03:00:00")) + .build(); + + // make sure that compactionTask becomes ready first, then the indexTask becomes ready, then compactionTask runs + final CountDownLatch indexTaskReadyLatch = new CountDownLatch(1); + final CountDownLatch compactionTaskStartLatch = new CountDownLatch(1); + final Future>> compactionFuture = exec.submit( + () -> { + final Pair> pair = runTask( + compactionTask, + indexTaskReadyLatch, + compactionTaskStartLatch + ); + return pair; + } + ); + + final Future>> indexFuture = exec.submit( + () -> { + indexTaskReadyLatch.await(); + return runIndexTask(compactionTaskStartLatch, null, false); + } + ); + + Assert.assertTrue(indexFuture.get().lhs.isSuccess()); + + List segments = indexFuture.get().rhs; + Assert.assertEquals(6, segments.size()); + + for (int i = 0; i < 6; i++) { + Assert.assertEquals( + Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i / 2, i / 2 + 1), + segments.get(i).getInterval() + ); + if (lockGranularity == LockGranularity.SEGMENT) { + Assert.assertEquals( + new NumberedOverwriteShardSpec( + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID + i % 2, + 0, + 2, + (short) 1, + (short) 2 + ), + segments.get(i).getShardSpec() + ); + } else { + Assert.assertEquals(new NumberedShardSpec(i % 2, 0), segments.get(i).getShardSpec()); + } + } + + final Pair> compactionResult = compactionFuture.get(); + Assert.assertEquals(TaskState.FAILED, compactionResult.lhs.getStatusCode()); + } + private Pair> runIndexTask() throws Exception + { + return runIndexTask(null, null, false); + } + + private Pair> runAppendTask() throws Exception + { + return runIndexTask(null, null, true); + } + + private Pair> runIndexTask( + @Nullable CountDownLatch readyLatchToCountDown, + @Nullable CountDownLatch latchToAwaitBeforeRun, + boolean appendToExisting + ) throws Exception { File tmpDir = temporaryFolder.newFolder(); File tmpFile = File.createTempFile("druid", "index", tmpDir); @@ -287,7 +628,8 @@ private Pair> runIndexTask() throws Exception IndexTask indexTask = new IndexTask( null, null, - createIngestionSpec( + IndexTaskTest.createIngestionSpec( + getObjectMapper(), tmpDir, DEFAULT_PARSE_SPEC, new UniformGranularitySpec( @@ -296,7 +638,7 @@ private Pair> runIndexTask() throws Exception null ), IndexTaskTest.createTuningConfig(2, 2, null, 2L, null, null, false, true), - false + appendToExisting ), null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, @@ -304,14 +646,23 @@ private Pair> runIndexTask() throws Exception rowIngestionMetersFactory ); - return runTask(indexTask); + return runTask(indexTask, readyLatchToCountDown, latchToAwaitBeforeRun); } private Pair> runTask(Task task) throws Exception + { + return runTask(task, null, null); + } + + private Pair> runTask( + Task task, + @Nullable CountDownLatch readyLatchToCountDown, + @Nullable CountDownLatch latchToAwaitBeforeRun + ) throws Exception { getLockbox().add(task); getTaskStorage().insert(task, TaskStatus.running(task.getId())); - final LocalTaskActionClient actionClient = createActionClient(task); + final TestLocalTaskActionClient actionClient = createActionClient(task); final File deepStorageDir = temporaryFolder.newFolder(); final ObjectMapper objectMapper = getObjectMapper(); @@ -320,26 +671,6 @@ private Pair> runTask(Task task) throws Exception ); objectMapper.registerSubtypes(LocalDataSegmentPuller.class); - final List segments = new ArrayList<>(); - final DataSegmentPusher pusher = new LocalDataSegmentPusher( - new LocalDataSegmentPusherConfig() - { - @Override - public File getStorageDirectory() - { - return deepStorageDir; - } - } - ) - { - @Override - public DataSegment push(File file, DataSegment segment, boolean useUniquePath) throws IOException - { - segments.add(segment); - return super.push(file, segment, useUniquePath); - } - }; - final SegmentLoader loader = new SegmentLoaderLocalCacheManager( getIndexIO(), new SegmentLoaderConfig() { @@ -356,7 +687,7 @@ public List getLocations() null, actionClient, null, - pusher, + new LocalDataSegmentPusher(new LocalDataSegmentPusherConfig()), new NoopDataSegmentKiller(), null, null, @@ -381,13 +712,21 @@ public List getLocations() new NoopTestTaskFileWriter() ); + task.addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, lockGranularity == LockGranularity.TIME_CHUNK); if (task.isReady(box.getTaskActionClient())) { + if (readyLatchToCountDown != null) { + readyLatchToCountDown.countDown(); + } + if (latchToAwaitBeforeRun != null) { + latchToAwaitBeforeRun.await(); + } TaskStatus status = task.run(box); shutdownTask(task); + final List segments = new ArrayList<>(actionClient.getPublishedSegments()); Collections.sort(segments); return Pair.of(status, segments); } else { - throw new ISE("task is not ready"); + throw new ISE("task[%s] is not ready", task.getId()); } } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index bb6993848c94..249404648ff4 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -1011,7 +1011,7 @@ private static List getDimensionSchema(DimensionSchema mixedTyp ); } - private static void assertIngestionSchema( + private void assertIngestionSchema( List ingestionSchemas, List expectedDimensionsSpecs, List expectedMetricsSpec, @@ -1055,7 +1055,7 @@ private static void assertIngestionSchema( ); } - private static void assertIngestionSchema( + private void assertIngestionSchema( List ingestionSchemas, List expectedDimensionsSpecs, List expectedMetricsSpec, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/HadoopTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/HadoopTaskTest.java index 64d3dd80113a..42c864163587 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/HadoopTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/HadoopTaskTest.java @@ -27,14 +27,20 @@ import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.config.TaskConfig; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.timeline.DataSegment; import org.apache.hadoop.yarn.util.ApplicationClassLoader; import org.easymock.EasyMock; +import org.joda.time.Interval; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import javax.annotation.Nullable; import java.net.URLClassLoader; +import java.util.Collections; +import java.util.List; public class HadoopTaskTest { @@ -63,6 +69,31 @@ public boolean isReady(TaskActionClient taskActionClient) return false; } + @Override + public boolean requireLockExistingSegments() + { + return true; + } + + @Override + public List findSegmentsToLock(TaskActionClient taskActionClient, List intervals) + { + return Collections.emptyList(); + } + + @Override + public boolean isPerfectRollup() + { + return true; + } + + @Nullable + @Override + public Granularity getSegmentGranularity() + { + return null; + } + @Override public TaskStatus run(TaskToolbox toolbox) { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java index a8f02e603ca8..45ced1688e13 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java @@ -38,30 +38,18 @@ import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.IngestionStatsAndErrorsTaskReportData; -import org.apache.druid.indexing.common.TaskLock; -import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.TaskReport; -import org.apache.druid.indexing.common.TaskReportFileWriter; -import org.apache.druid.indexing.common.TaskToolbox; -import org.apache.druid.indexing.common.TestUtils; -import org.apache.druid.indexing.common.actions.LockAcquireAction; -import org.apache.druid.indexing.common.actions.LockListAction; -import org.apache.druid.indexing.common.actions.LockTryAcquireAction; import org.apache.druid.indexing.common.actions.SegmentAllocateAction; -import org.apache.druid.indexing.common.actions.SegmentTransactionalInsertAction; -import org.apache.druid.indexing.common.actions.TaskAction; -import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.stats.RowIngestionMeters; import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; import org.apache.druid.indexing.common.task.IndexTask.IndexIngestionSpec; import org.apache.druid.indexing.common.task.IndexTask.IndexTuningConfig; -import org.apache.druid.indexing.overlord.SegmentPublishResult; -import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.java.util.common.guava.Comparators; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -71,7 +59,6 @@ import org.apache.druid.segment.Cursor; import org.apache.druid.segment.DimensionSelector; import org.apache.druid.segment.IndexIO; -import org.apache.druid.segment.IndexMergerV9; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.QueryableIndexStorageAdapter; import org.apache.druid.segment.VirtualColumns; @@ -79,15 +66,10 @@ import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import org.apache.druid.segment.indexing.granularity.GranularitySpec; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; -import org.apache.druid.segment.loading.DataSegmentKiller; -import org.apache.druid.segment.loading.DataSegmentPusher; -import org.apache.druid.segment.loading.LocalDataSegmentPusher; -import org.apache.druid.segment.loading.LocalDataSegmentPusherConfig; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.loading.SegmentLoaderConfig; import org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager; import org.apache.druid.segment.loading.StorageLocationConfig; -import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.segment.realtime.firehose.LocalFirehoseFactory; import org.apache.druid.segment.realtime.firehose.WindowedStorageAdapter; import org.apache.druid.segment.transform.ExpressionTransform; @@ -95,16 +77,18 @@ import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec; +import org.apache.druid.timeline.partition.NumberedOverwriteShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; -import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.timeline.partition.PartitionIds; import org.joda.time.Interval; -import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import javax.annotation.Nullable; import java.io.BufferedWriter; @@ -119,7 +103,8 @@ import java.util.Map; import java.util.Set; -public class IndexTaskTest +@RunWith(Parameterized.class) +public class IndexTaskTest extends IngestionTestBase { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -144,58 +129,35 @@ public class IndexTaskTest 0 ); - private DataSegmentPusher pusher; - private SegmentLoader segmentLoader; - private List segments; + @Parameterized.Parameters(name = "{0}") + public static Iterable constructorFeeder() + { + return ImmutableList.of( + new Object[]{LockGranularity.TIME_CHUNK}, + new Object[]{LockGranularity.SEGMENT} + ); + } private static final IndexSpec indexSpec = new IndexSpec(); private final ObjectMapper jsonMapper; - private IndexMergerV9 indexMergerV9; - private IndexIO indexIO; - private volatile int segmentAllocatePartitionCounter; - private File reportsFile; - private RowIngestionMetersFactory rowIngestionMetersFactory; + private final IndexIO indexIO; + private final RowIngestionMetersFactory rowIngestionMetersFactory; + private final LockGranularity lockGranularity; + private SegmentLoader segmentLoader; + private TestTaskRunner taskRunner; - public IndexTaskTest() + public IndexTaskTest(LockGranularity lockGranularity) { - TestUtils testUtils = new TestUtils(); - jsonMapper = testUtils.getTestObjectMapper(); - - indexMergerV9 = testUtils.getTestIndexMergerV9(); - indexIO = testUtils.getTestIndexIO(); - rowIngestionMetersFactory = testUtils.getRowIngestionMetersFactory(); + this.jsonMapper = getObjectMapper(); + this.indexIO = getIndexIO(); + this.rowIngestionMetersFactory = getRowIngestionMetersFactory(); + this.lockGranularity = lockGranularity; } @Before public void setup() throws IOException { - reportsFile = temporaryFolder.newFile( - StringUtils.format("IndexTaskTestReports-%s.json", System.currentTimeMillis()) - ); - - final File deepStorageDir = temporaryFolder.newFolder(); final File cacheDir = temporaryFolder.newFolder(); - - pusher = new LocalDataSegmentPusher( - new LocalDataSegmentPusherConfig() - { - @Override - public File getStorageDirectory() - { - return deepStorageDir; - } - } - ) - { - @Override - public DataSegment push(final File dataSegmentFile, final DataSegment segment, final boolean useUniquePath) - throws IOException - { - final DataSegment returnSegment = super.push(dataSegmentFile, segment, useUniquePath); - segments.add(returnSegment); - return returnSegment; - } - }; segmentLoader = new SegmentLoaderLocalCacheManager( indexIO, new SegmentLoaderConfig() @@ -210,14 +172,7 @@ public List getLocations() }, jsonMapper ); - segments = new ArrayList<>(); - - } - - @After - public void teardown() - { - reportsFile.delete(); + taskRunner = new TestTaskRunner(); } @Test @@ -237,6 +192,7 @@ public void testDeterminePartitions() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, null, @@ -283,6 +239,7 @@ public void testTransformSpec() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, new TransformSpec( @@ -330,6 +287,7 @@ public void testWithArbitraryGranularity() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, new ArbitraryGranularitySpec( @@ -366,6 +324,7 @@ public void testIntervalBucketing() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, new UniformGranularitySpec( @@ -403,6 +362,7 @@ public void testNumShardsProvided() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, null, @@ -441,6 +401,7 @@ public void testNumShardsAndPartitionDimensionsProvided() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, null, @@ -453,7 +414,7 @@ public void testNumShardsAndPartitionDimensionsProvided() throws Exception rowIngestionMetersFactory ); - runTask(indexTask); + final List segments = runTask(indexTask).rhs; Assert.assertEquals(2, segments.size()); @@ -502,7 +463,6 @@ public void testNumShardsAndPartitionDimensionsProvided() throws Exception @Test public void testAppendToExisting() throws Exception { - segmentAllocatePartitionCounter = 0; File tmpDir = temporaryFolder.newFolder(); File tmpFile = File.createTempFile("druid", "index", tmpDir); @@ -516,6 +476,7 @@ public void testAppendToExisting() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, null, @@ -532,7 +493,7 @@ public void testAppendToExisting() throws Exception final List segments = runTask(indexTask).rhs; - Assert.assertEquals(2, segmentAllocatePartitionCounter); + Assert.assertEquals(2, taskRunner.getTaskActionClient().getActionCount(SegmentAllocateAction.class)); Assert.assertEquals(2, segments.size()); Assert.assertEquals("test", segments.get(0).getDataSource()); @@ -562,6 +523,7 @@ public void testIntervalNotSpecified() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, new UniformGranularitySpec( @@ -614,6 +576,7 @@ public void testCSVFileWithHeader() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, new CSVParseSpec( new TimestampSpec( @@ -666,6 +629,7 @@ public void testCSVFileWithHeaderColumnOverride() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, new CSVParseSpec( new TimestampSpec( @@ -724,6 +688,7 @@ public void testWithSmallMaxTotalRows() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, new UniformGranularitySpec( @@ -768,6 +733,7 @@ public void testPerfectRollup() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, new UniformGranularitySpec( @@ -812,6 +778,7 @@ public void testBestEffortRollup() throws Exception null, null, createIngestionSpec( + jsonMapper, tmpDir, null, new UniformGranularitySpec( @@ -833,9 +800,9 @@ public void testBestEffortRollup() throws Exception Assert.assertEquals(5, segments.size()); + final Interval expectedInterval = Intervals.of("2014-01-01T00:00:00.000Z/2014-01-02T00:00:00.000Z"); for (int i = 0; i < 5; i++) { final DataSegment segment = segments.get(i); - final Interval expectedInterval = Intervals.of("2014-01-01T00:00:00.000Z/2014-01-02T00:00:00.000Z"); Assert.assertEquals("test", segment.getDataSource()); Assert.assertEquals(expectedInterval, segment.getInterval()); @@ -875,6 +842,7 @@ public void testIgnoreParseException() throws Exception // GranularitySpec.intervals and numShards must be null to verify reportParseException=false is respected both in // IndexTask.determineShardSpecs() and IndexTask.generateAndPublishSegments() final IndexIngestionSpec parseExceptionIgnoreSpec = createIngestionSpec( + jsonMapper, tmpDir, new CSVParseSpec( new TimestampSpec( @@ -928,6 +896,7 @@ public void testReportParseException() throws Exception } final IndexIngestionSpec parseExceptionIgnoreSpec = createIngestionSpec( + jsonMapper, tmpDir, new CSVParseSpec( new TimestampSpec( @@ -984,11 +953,15 @@ public void testMultipleParseExceptionsSuccess() throws Exception try (BufferedWriter writer = Files.newWriter(tmpFile, StandardCharsets.UTF_8)) { writer.write("{\"time\":\"unparseable\",\"dim\":\"a\",\"dimLong\":2,\"dimFloat\":3.0,\"val\":1}\n"); // unparseable time writer.write("{\"time\":\"2014-01-01T00:00:10Z\",\"dim\":\"a\",\"dimLong\":2,\"dimFloat\":3.0,\"val\":1}\n"); // valid row - writer.write("{\"time\":\"2014-01-01T00:00:10Z\",\"dim\":\"b\",\"dimLong\":\"notnumber\",\"dimFloat\":3.0,\"val\":1}\n"); // row with invalid long dimension - writer.write("{\"time\":\"2014-01-01T00:00:10Z\",\"dim\":\"b\",\"dimLong\":2,\"dimFloat\":\"notnumber\",\"val\":1}\n"); // row with invalid float dimension - writer.write("{\"time\":\"2014-01-01T00:00:10Z\",\"dim\":\"b\",\"dimLong\":2,\"dimFloat\":4.0,\"val\":\"notnumber\"}\n"); // row with invalid metric + writer.write( + "{\"time\":\"2014-01-01T00:00:10Z\",\"dim\":\"b\",\"dimLong\":\"notnumber\",\"dimFloat\":3.0,\"val\":1}\n"); // row with invalid long dimension + writer.write( + "{\"time\":\"2014-01-01T00:00:10Z\",\"dim\":\"b\",\"dimLong\":2,\"dimFloat\":\"notnumber\",\"val\":1}\n"); // row with invalid float dimension + writer.write( + "{\"time\":\"2014-01-01T00:00:10Z\",\"dim\":\"b\",\"dimLong\":2,\"dimFloat\":4.0,\"val\":\"notnumber\"}\n"); // row with invalid metric writer.write("{\"time\":9.0x,\"dim\":\"a\",\"dimLong\":2,\"dimFloat\":3.0,\"val\":1}\n"); // invalid JSON - writer.write("{\"time\":\"3014-03-01T00:00:10Z\",\"dim\":\"outsideofinterval\",\"dimLong\":2,\"dimFloat\":3.0,\"val\":1}\n"); // thrown away + writer.write( + "{\"time\":\"3014-03-01T00:00:10Z\",\"dim\":\"outsideofinterval\",\"dimLong\":2,\"dimFloat\":3.0,\"val\":1}\n"); // thrown away writer.write("{\"time\":\"99999999999-01-01T00:00:10Z\",\"dim\":\"b\",\"dimLong\":2,\"dimFloat\":3.0,\"val\":1}\n"); // unparseable time writer.write("this is not JSON\n"); // invalid JSON } @@ -1017,6 +990,7 @@ public void testMultipleParseExceptionsSuccess() throws Exception ); final IndexIngestionSpec parseExceptionIgnoreSpec = createIngestionSpec( + jsonMapper, tmpDir, new JSONParseSpec( new TimestampSpec( @@ -1140,6 +1114,7 @@ public void testMultipleParseExceptionsFailure() throws Exception ); final IndexIngestionSpec parseExceptionIgnoreSpec = createIngestionSpec( + jsonMapper, tmpDir, new CSVParseSpec( new TimestampSpec( @@ -1256,6 +1231,7 @@ public void testMultipleParseExceptionsFailureAtDeterminePartitions() throws Exc ); final IndexIngestionSpec parseExceptionIgnoreSpec = createIngestionSpec( + jsonMapper, tmpDir, new CSVParseSpec( new TimestampSpec( @@ -1359,6 +1335,7 @@ public void testCsvWithHeaderOfEmptyColumns() throws Exception } final IndexIngestionSpec parseExceptionIgnoreSpec = createIngestionSpec( + jsonMapper, tmpDir, new CSVParseSpec( new TimestampSpec( @@ -1429,6 +1406,7 @@ public void testCsvWithHeaderOfEmptyTimestamp() throws Exception } final IndexIngestionSpec parseExceptionIgnoreSpec = createIngestionSpec( + jsonMapper, tmpDir, new CSVParseSpec( new TimestampSpec( @@ -1478,178 +1456,133 @@ public void testCsvWithHeaderOfEmptyTimestamp() throws Exception Assert.assertEquals(expectedUnparseables, reportData.getUnparseableEvents()); } - public static void checkTaskStatusErrorMsgForParseExceptionsExceeded(TaskStatus status) + @Test + public void testOverwriteWithSameSegmentGranularity() throws Exception { - // full stacktrace will be too long and make tests brittle (e.g. if line # changes), just match the main message - Assert.assertTrue(status.getErrorMsg().contains("Max parse exceptions exceeded, terminating task...")); - } + final File tmpDir = temporaryFolder.newFolder(); + final File tmpFile = File.createTempFile("druid", "index", tmpDir); - private Pair> runTask(IndexTask indexTask) throws Exception - { - final TaskActionClient actionClient = new TaskActionClient() - { - @Override - public RetType submit(TaskAction taskAction) - { - if (taskAction instanceof LockListAction) { - return (RetType) Collections.singletonList( - new TaskLock( - TaskLockType.EXCLUSIVE, - "", - "", - Intervals.of("2014/P1Y"), DateTimes.nowUtc().toString(), - Tasks.DEFAULT_BATCH_INDEX_TASK_PRIORITY - ) - ); - } + populateRollupTestData(tmpFile); - if (taskAction instanceof LockAcquireAction) { - return (RetType) new TaskLock( - TaskLockType.EXCLUSIVE, - "groupId", - "test", - ((LockAcquireAction) taskAction).getInterval(), - DateTimes.nowUtc().toString(), - Tasks.DEFAULT_BATCH_INDEX_TASK_PRIORITY - ); - } + for (int i = 0; i < 2; i++) { + final IndexTask indexTask = new IndexTask( + null, + null, + createIngestionSpec( + jsonMapper, + tmpDir, + null, + new UniformGranularitySpec( + Granularities.DAY, + Granularities.DAY, + true, + null + ), + createTuningConfig(3, 2, null, 2L, null, null, false, true), + false + ), + null, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + null, + rowIngestionMetersFactory + ); - if (taskAction instanceof LockTryAcquireAction) { - return (RetType) new TaskLock( - TaskLockType.EXCLUSIVE, - "groupId", - "test", - ((LockTryAcquireAction) taskAction).getInterval(), - DateTimes.nowUtc().toString(), - Tasks.DEFAULT_BATCH_INDEX_TASK_PRIORITY - ); - } + final List segments = runTask(indexTask).rhs; - if (taskAction instanceof SegmentTransactionalInsertAction) { - return (RetType) SegmentPublishResult.ok(((SegmentTransactionalInsertAction) taskAction).getSegments()); - } + Assert.assertEquals(5, segments.size()); - if (taskAction instanceof SegmentAllocateAction) { - SegmentAllocateAction action = (SegmentAllocateAction) taskAction; - Interval interval = action.getPreferredSegmentGranularity().bucket(action.getTimestamp()); - ShardSpec shardSpec = new NumberedShardSpec(segmentAllocatePartitionCounter++, 0); - return (RetType) new SegmentIdWithShardSpec(action.getDataSource(), interval, "latestVersion", shardSpec); + final Interval expectedInterval = Intervals.of("2014-01-01T00:00:00.000Z/2014-01-02T00:00:00.000Z"); + for (int j = 0; j < 5; j++) { + final DataSegment segment = segments.get(j); + Assert.assertEquals("test", segment.getDataSource()); + Assert.assertEquals(expectedInterval, segment.getInterval()); + if (i == 0) { + Assert.assertEquals(NumberedShardSpec.class, segment.getShardSpec().getClass()); + Assert.assertEquals(j, segment.getShardSpec().getPartitionNum()); + } else { + if (lockGranularity == LockGranularity.SEGMENT) { + Assert.assertEquals(NumberedOverwriteShardSpec.class, segment.getShardSpec().getClass()); + final NumberedOverwriteShardSpec numberedOverwriteShardSpec = + (NumberedOverwriteShardSpec) segment.getShardSpec(); + Assert.assertEquals( + j + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID, + numberedOverwriteShardSpec.getPartitionNum() + ); + Assert.assertEquals(1, numberedOverwriteShardSpec.getMinorVersion()); + Assert.assertEquals(5, numberedOverwriteShardSpec.getAtomicUpdateGroupSize()); + Assert.assertEquals(0, numberedOverwriteShardSpec.getStartRootPartitionId()); + Assert.assertEquals(5, numberedOverwriteShardSpec.getEndRootPartitionId()); + } else { + Assert.assertEquals(NumberedShardSpec.class, segment.getShardSpec().getClass()); + final NumberedShardSpec numberedShardSpec = (NumberedShardSpec) segment.getShardSpec(); + Assert.assertEquals(j, numberedShardSpec.getPartitionNum()); + } } - - return null; } - }; + } + } - final DataSegmentKiller killer = new DataSegmentKiller() - { - @Override - public void kill(DataSegment segment) - { + @Test + public void testOverwriteWithDifferentSegmentGranularity() throws Exception + { + final File tmpDir = temporaryFolder.newFolder(); + final File tmpFile = File.createTempFile("druid", "index", tmpDir); - } + populateRollupTestData(tmpFile); - @Override - public void killAll() - { + for (int i = 0; i < 2; i++) { + final Granularity segmentGranularity = i == 0 ? Granularities.DAY : Granularities.MONTH; + final IndexTask indexTask = new IndexTask( + null, + null, + createIngestionSpec( + jsonMapper, + tmpDir, + null, + new UniformGranularitySpec( + segmentGranularity, + Granularities.DAY, + true, + null + ), + createTuningConfig(3, 2, null, 2L, null, null, false, true), + false + ), + null, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + null, + rowIngestionMetersFactory + ); - } - }; + final List segments = runTask(indexTask).rhs; - final TaskToolbox box = new TaskToolbox( - null, - actionClient, - null, - pusher, - killer, - null, - null, - null, - null, - null, - null, - null, - null, - null, - jsonMapper, - temporaryFolder.newFolder(), - indexIO, - null, - null, - null, - indexMergerV9, - null, - null, - null, - null, - new TaskReportFileWriter(reportsFile) - ); - - indexTask.isReady(box.getTaskActionClient()); - TaskStatus status = indexTask.run(box); + Assert.assertEquals(5, segments.size()); - segments.sort((s1, s2) -> { - final int comp = Comparators.intervalsByStartThenEnd().compare(s1.getInterval(), s2.getInterval()); - if (comp != 0) { - return comp; + final Interval expectedInterval = i == 0 + ? Intervals.of("2014-01-01/2014-01-02") + : Intervals.of("2014-01-01/2014-02-01"); + for (int j = 0; j < 5; j++) { + final DataSegment segment = segments.get(j); + Assert.assertEquals("test", segment.getDataSource()); + Assert.assertEquals(expectedInterval, segment.getInterval()); + Assert.assertEquals(NumberedShardSpec.class, segment.getShardSpec().getClass()); + Assert.assertEquals(j, segment.getShardSpec().getPartitionNum()); } - //noinspection SubtractionInCompareTo - return s1.getShardSpec().getPartitionNum() - s2.getShardSpec().getPartitionNum(); - }); - - return Pair.of(status, segments); + } } - private IndexTask.IndexIngestionSpec createIngestionSpec( - File baseDir, - ParseSpec parseSpec, - GranularitySpec granularitySpec, - IndexTuningConfig tuningConfig, - boolean appendToExisting - ) + public static void checkTaskStatusErrorMsgForParseExceptionsExceeded(TaskStatus status) { - return createIngestionSpec(baseDir, parseSpec, TransformSpec.NONE, granularitySpec, tuningConfig, appendToExisting); + // full stacktrace will be too long and make tests brittle (e.g. if line # changes), just match the main message + Assert.assertTrue(status.getErrorMsg().contains("Max parse exceptions exceeded, terminating task...")); } - private IndexTask.IndexIngestionSpec createIngestionSpec( - File baseDir, - ParseSpec parseSpec, - TransformSpec transformSpec, - GranularitySpec granularitySpec, - IndexTuningConfig tuningConfig, - boolean appendToExisting - ) + private Pair> runTask(IndexTask task) throws Exception { - return new IndexTask.IndexIngestionSpec( - new DataSchema( - "test", - jsonMapper.convertValue( - new StringInputRowParser( - parseSpec != null ? parseSpec : DEFAULT_PARSE_SPEC, - null - ), - Map.class - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - granularitySpec != null ? granularitySpec : new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2014/2015")) - ), - transformSpec, - jsonMapper - ), - new IndexTask.IndexIOConfig( - new LocalFirehoseFactory( - baseDir, - "druid*", - null - ), - appendToExisting - ), - tuningConfig - ); + task.addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, lockGranularity == LockGranularity.TIME_CHUNK); + final TaskStatus status = taskRunner.run(task).get(); + final List segments = taskRunner.getPublishedSegments(); + return Pair.of(status, segments); } private static IndexTuningConfig createTuningConfigWithMaxRowsPerSegment( @@ -1725,7 +1658,7 @@ static IndexTuningConfig createTuningConfig( private IngestionStatsAndErrorsTaskReportData getTaskReportData() throws IOException { Map taskReports = jsonMapper.readValue( - reportsFile, + taskRunner.getTaskReportsFile(), new TypeReference>() { } @@ -1734,4 +1667,67 @@ private IngestionStatsAndErrorsTaskReportData getTaskReportData() throws IOExcep taskReports ); } + + public static IndexTask.IndexIngestionSpec createIngestionSpec( + ObjectMapper objectMapper, + File baseDir, + ParseSpec parseSpec, + GranularitySpec granularitySpec, + IndexTuningConfig tuningConfig, + boolean appendToExisting + ) + { + return createIngestionSpec( + objectMapper, + baseDir, + parseSpec, + TransformSpec.NONE, + granularitySpec, + tuningConfig, + appendToExisting + ); + } + + public static IndexTask.IndexIngestionSpec createIngestionSpec( + ObjectMapper objectMapper, + File baseDir, + ParseSpec parseSpec, + TransformSpec transformSpec, + GranularitySpec granularitySpec, + IndexTuningConfig tuningConfig, + boolean appendToExisting + ) + { + return new IndexTask.IndexIngestionSpec( + new DataSchema( + "test", + objectMapper.convertValue( + new StringInputRowParser( + parseSpec != null ? parseSpec : DEFAULT_PARSE_SPEC, + null + ), + Map.class + ), + new AggregatorFactory[]{ + new LongSumAggregatorFactory("val", "val") + }, + granularitySpec != null ? granularitySpec : new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + Collections.singletonList(Intervals.of("2014/2015")) + ), + transformSpec, + objectMapper + ), + new IndexTask.IndexIOConfig( + new LocalFirehoseFactory( + baseDir, + "druid*", + null + ), + appendToExisting + ), + tuningConfig + ); + } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java index c70a71a939b5..15e9aa5e12ee 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java @@ -20,44 +20,65 @@ package org.apache.druid.indexing.common.task; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.data.input.impl.ParseSpec; -import org.apache.druid.data.input.impl.StringInputRowParser; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.indexer.TaskStatus; +import org.apache.druid.indexing.common.TaskReportFileWriter; +import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.TestUtils; import org.apache.druid.indexing.common.actions.LocalTaskActionClient; +import org.apache.druid.indexing.common.actions.SegmentInsertAction; +import org.apache.druid.indexing.common.actions.SegmentTransactionalInsertAction; +import org.apache.druid.indexing.common.actions.TaskAction; import org.apache.druid.indexing.common.actions.TaskActionToolbox; import org.apache.druid.indexing.common.actions.TaskAuditLogConfig; import org.apache.druid.indexing.common.config.TaskStorageConfig; -import org.apache.druid.indexing.common.task.IndexTask.IndexTuningConfig; +import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage; +import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.TaskLockbox; +import org.apache.druid.indexing.overlord.TaskRunner; +import org.apache.druid.indexing.overlord.TaskRunnerListener; +import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.indexing.overlord.TaskStorage; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.indexing.overlord.autoscaling.ScalingStats; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.metadata.EntryExistsException; import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator; +import org.apache.druid.metadata.MetadataSegmentManager; +import org.apache.druid.metadata.MetadataSegmentManagerConfig; import org.apache.druid.metadata.SQLMetadataConnector; +import org.apache.druid.metadata.SQLMetadataSegmentManager; import org.apache.druid.metadata.TestDerbyConnector; -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.IndexMergerV9; -import org.apache.druid.segment.indexing.DataSchema; -import org.apache.druid.segment.indexing.granularity.GranularitySpec; -import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; -import org.apache.druid.segment.realtime.firehose.LocalFirehoseFactory; -import org.apache.druid.segment.transform.TransformSpec; +import org.apache.druid.segment.loading.LocalDataSegmentPusher; +import org.apache.druid.segment.loading.LocalDataSegmentPusherConfig; +import org.apache.druid.segment.loading.NoopDataSegmentKiller; import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.timeline.DataSegment; +import org.junit.After; import org.junit.Before; import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.Map; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Executor; public abstract class IngestionTestBase { - public static final String DATA_SOURCE = "test"; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Rule public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new TestDerbyConnector.DerbyConnectorRule(); @@ -66,25 +87,41 @@ public abstract class IngestionTestBase private final ObjectMapper objectMapper = testUtils.getTestObjectMapper(); private TaskStorage taskStorage; private IndexerSQLMetadataStorageCoordinator storageCoordinator; + private MetadataSegmentManager segmentManager; private TaskLockbox lockbox; @Before - public void setUp() + public void setUp() throws IOException { + temporaryFolder.create(); + final SQLMetadataConnector connector = derbyConnectorRule.getConnector(); connector.createTaskTables(); + connector.createSegmentTable(); taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null)); storageCoordinator = new IndexerSQLMetadataStorageCoordinator( objectMapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnectorRule.getConnector() ); - lockbox = new TaskLockbox(taskStorage); + segmentManager = new SQLMetadataSegmentManager( + objectMapper, + MetadataSegmentManagerConfig::new, + derbyConnectorRule.metadataTablesConfigSupplier(), + derbyConnectorRule.getConnector() + ); + lockbox = new TaskLockbox(taskStorage, storageCoordinator); } - public LocalTaskActionClient createActionClient(Task task) + @After + public void tearDown() { - return new LocalTaskActionClient(task, taskStorage, createTaskActionToolbox(), new TaskAuditLogConfig(false)); + temporaryFolder.delete(); + } + + public TestLocalTaskActionClient createActionClient(Task task) + { + return new TestLocalTaskActionClient(task); } public void prepareTaskForLocking(Task task) throws EntryExistsException @@ -108,6 +145,16 @@ public TaskStorage getTaskStorage() return taskStorage; } + public IndexerMetadataStorageCoordinator getMetadataStorageCoordinator() + { + return storageCoordinator; + } + + public MetadataSegmentManager getMetadataSegmentManager() + { + return segmentManager; + } + public TaskLockbox getLockbox() { return lockbox; @@ -118,6 +165,11 @@ public IndexerSQLMetadataStorageCoordinator getStorageCoordinator() return storageCoordinator; } + public RowIngestionMetersFactory getRowIngestionMetersFactory() + { + return testUtils.getRowIngestionMetersFactory(); + } + public TaskActionToolbox createTaskActionToolbox() { storageCoordinator.start(); @@ -140,53 +192,167 @@ public IndexMergerV9 getIndexMerger() return testUtils.getTestIndexMergerV9(); } - public IndexTask.IndexIngestionSpec createIngestionSpec( - File baseDir, - ParseSpec parseSpec, - GranularitySpec granularitySpec, - IndexTuningConfig tuningConfig, - boolean appendToExisting - ) - { - return createIngestionSpec(baseDir, parseSpec, TransformSpec.NONE, granularitySpec, tuningConfig, appendToExisting); - } - - public IndexTask.IndexIngestionSpec createIngestionSpec( - File baseDir, - ParseSpec parseSpec, - TransformSpec transformSpec, - GranularitySpec granularitySpec, - IndexTuningConfig tuningConfig, - boolean appendToExisting - ) - { - return new IndexTask.IndexIngestionSpec( - new DataSchema( - DATA_SOURCE, - objectMapper.convertValue( - new StringInputRowParser(parseSpec, null), - Map.class - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - granularitySpec != null ? granularitySpec : new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2014/2015")) - ), - transformSpec, - objectMapper - ), - new IndexTask.IndexIOConfig( - new LocalFirehoseFactory( - baseDir, - "druid*", - null - ), - appendToExisting - ), - tuningConfig - ); + public class TestLocalTaskActionClient extends LocalTaskActionClient + { + private final Set publishedSegments = new HashSet<>(); + + private TestLocalTaskActionClient(Task task) + { + super(task, taskStorage, createTaskActionToolbox(), new TaskAuditLogConfig(false)); + } + + @Override + public RetType submit(TaskAction taskAction) + { + final RetType result = super.submit(taskAction); + if (taskAction instanceof SegmentTransactionalInsertAction) { + publishedSegments.addAll(((SegmentTransactionalInsertAction) taskAction).getSegments()); + } else if (taskAction instanceof SegmentInsertAction) { + publishedSegments.addAll(((SegmentInsertAction) taskAction).getSegments()); + } + return result; + } + + public Set getPublishedSegments() + { + return publishedSegments; + } + } + + public class TestTaskRunner implements TaskRunner + { + private TestLocalTaskActionClient taskActionClient; + private File taskReportsFile; + + @Override + public List>> restore() + { + throw new UnsupportedOperationException(); + } + + @Override + public void start() + { + throw new UnsupportedOperationException(); + } + + @Override + public void registerListener(TaskRunnerListener listener, Executor executor) + { + throw new UnsupportedOperationException(); + } + + @Override + public void unregisterListener(String listenerId) + { + throw new UnsupportedOperationException(); + } + + public TestLocalTaskActionClient getTaskActionClient() + { + return taskActionClient; + } + + public File getTaskReportsFile() + { + return taskReportsFile; + } + + public List getPublishedSegments() + { + final List segments = new ArrayList<>(taskActionClient.getPublishedSegments()); + Collections.sort(segments); + return segments; + } + + @Override + public ListenableFuture run(Task task) + { + try { + lockbox.add(task); + taskStorage.insert(task, TaskStatus.running(task.getId())); + taskActionClient = createActionClient(task); + taskReportsFile = temporaryFolder.newFile( + StringUtils.format("ingestionTestBase-%s.json", System.currentTimeMillis()) + ); + + final TaskToolbox box = new TaskToolbox( + null, + taskActionClient, + null, + new LocalDataSegmentPusher(new LocalDataSegmentPusherConfig()), + new NoopDataSegmentKiller(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + objectMapper, + temporaryFolder.newFolder(), + getIndexIO(), + null, + null, + null, + getIndexMerger(), + null, + null, + null, + null, + new TaskReportFileWriter(taskReportsFile) + ); + + if (task.isReady(box.getTaskActionClient())) { + return Futures.immediateFuture(task.run(box)); + } else { + throw new ISE("task is not ready"); + } + } + catch (Exception e) { + throw new RuntimeException(e); + } + finally { + lockbox.remove(task); + } + } + + @Override + public void shutdown(String taskid, String reason) + { + throw new UnsupportedOperationException(); + } + + @Override + public void stop() + { + throw new UnsupportedOperationException(); + } + + @Override + public Collection getRunningTasks() + { + throw new UnsupportedOperationException(); + } + + @Override + public Collection getPendingTasks() + { + throw new UnsupportedOperationException(); + } + + @Override + public Collection getKnownTasks() + { + throw new UnsupportedOperationException(); + } + + @Override + public Optional getScalingStats() + { + throw new UnsupportedOperationException(); + } } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillTaskTest.java new file mode 100644 index 000000000000..cc4334fc7622 --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillTaskTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.task; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.druid.indexer.TaskState; +import org.apache.druid.indexing.overlord.TaskRunner; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Set; + +public class KillTaskTest extends IngestionTestBase +{ + private static final String DATA_SOURCE = "dataSource"; + + private TaskRunner taskRunner; + + @Before + public void setup() + { + taskRunner = new TestTaskRunner(); + } + + @Test + public void testKill() throws Exception + { + final String version = DateTimes.nowUtc().toString(); + final Set segments = ImmutableSet.of( + newSegment(Intervals.of("2019-01-01/2019-02-01"), version), + newSegment(Intervals.of("2019-02-01/2019-03-01"), version), + newSegment(Intervals.of("2019-03-01/2019-04-01"), version), + newSegment(Intervals.of("2019-04-01/2019-05-01"), version) + ); + final Set announced = getMetadataStorageCoordinator().announceHistoricalSegments(segments); + + Assert.assertEquals(segments, announced); + + Assert.assertTrue( + getMetadataSegmentManager().markSegmentAsUnused( + newSegment(Intervals.of("2019-02-01/2019-03-01"), version).getId().toString() + ) + ); + Assert.assertTrue( + getMetadataSegmentManager().markSegmentAsUnused( + newSegment(Intervals.of("2019-03-01/2019-04-01"), version).getId().toString() + ) + ); + + final KillTask task = new KillTask(null, DATA_SOURCE, Intervals.of("2019-03-01/2019-04-01"), null); + + Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); + + final List unusedSegments = getMetadataStorageCoordinator().getUnusedSegmentsForInterval( + DATA_SOURCE, + Intervals.of("2019/2020") + ); + + Assert.assertEquals(ImmutableList.of(newSegment(Intervals.of("2019-02-01/2019-03-01"), version)), unusedSegments); + Assert.assertEquals( + ImmutableList.of( + newSegment(Intervals.of("2019-01-01/2019-02-01"), version), + newSegment(Intervals.of("2019-04-01/2019-05-01"), version) + ), + getMetadataStorageCoordinator().getUsedSegmentsForInterval(DATA_SOURCE, Intervals.of("2019/2020")) + ); + } + + private static DataSegment newSegment(Interval interval, String version) + { + return new DataSegment( + DATA_SOURCE, + interval, + version, + null, + null, + null, + null, + 9, + 10L + ); + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/RealtimeIndexTaskTest.java index 316327b7fb7d..2aa361bcce8a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -882,7 +882,7 @@ private TaskToolbox makeToolbox( ) { final TaskConfig taskConfig = new TaskConfig(directory.getPath(), null, null, 50000, null, true, null, null, null); - final TaskLockbox taskLockbox = new TaskLockbox(taskStorage); + final TaskLockbox taskLockbox = new TaskLockbox(taskStorage, mdc); try { taskStorage.insert(task, TaskStatus.running(task.getId())); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java index 8792956bc95a..2bc229d43ad8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java @@ -49,9 +49,9 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.segment.loading.DataSegmentKiller; import org.apache.druid.segment.loading.LocalDataSegmentPusher; import org.apache.druid.segment.loading.LocalDataSegmentPusherConfig; +import org.apache.druid.segment.loading.NoopDataSegmentKiller; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.segment.realtime.firehose.NoopChatHandlerProvider; import org.apache.druid.server.security.AllowAllAuthorizer; @@ -68,8 +68,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; @@ -230,18 +230,7 @@ public File getStorageDirectory() } } ), - new DataSegmentKiller() - { - @Override - public void kill(DataSegment segment) - { - } - - @Override - public void killAll() - { - } - }, + new NoopDataSegmentKiller(), null, null, null, @@ -375,9 +364,9 @@ public SegmentIdWithShardSpec allocateSegment(String supervisorTaskId, DateTime } @Override - public void report(String supervisorTaskId, List pushedSegments) + public void report(String supervisorTaskId, Set oldSegments, Set pushedSegments) { - supervisorTask.getRunner().collectReport(new PushedSegmentsReport(getSubtaskId(), pushedSegments)); + supervisorTask.getRunner().collectReport(new PushedSegmentsReport(getSubtaskId(), oldSegments, pushedSegments)); } } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskResourceTest.java index 04aa5a7a1b07..c0a38c54db6b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskResourceTest.java @@ -24,17 +24,14 @@ import org.apache.druid.client.indexing.IndexingServiceClient; import org.apache.druid.data.input.FiniteFirehoseFactory; import org.apache.druid.data.input.InputSplit; +import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.StringInputRowParser; import org.apache.druid.indexer.RunnerTaskState; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexer.TaskStatusPlus; -import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.TaskToolbox; -import org.apache.druid.indexing.common.actions.LockListAction; -import org.apache.druid.indexing.common.actions.SurrogateAction; -import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.task.AbstractTask; import org.apache.druid.indexing.common.task.IndexTaskClientFactory; import org.apache.druid.indexing.common.task.TaskResource; @@ -48,10 +45,11 @@ import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; +import org.apache.druid.segment.realtime.appenderator.SegmentAllocator; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; import org.joda.time.Interval; import org.junit.After; @@ -502,6 +500,7 @@ private class TestSupervisorTask extends TestParallelIndexSupervisorTask @Override ParallelIndexTaskRunner createRunner(TaskToolbox toolbox) { + setToolbox(toolbox); setRunner( new TestRunner( toolbox, @@ -544,7 +543,6 @@ ParallelIndexSubTaskSpec newTaskSpec(InputSplit split) supervisorTask.getId() + "_" + getAndIncrementNextSpecId(), supervisorTask.getGroupId(), supervisorTask, - this, new ParallelIndexIngestionSpec( getIngestionSchema().getDataSchema(), new ParallelIndexIOConfig( @@ -569,7 +567,6 @@ private class TestSubTaskSpec extends ParallelIndexSubTaskSpec String id, String groupId, ParallelIndexSupervisorTask supervisorTask, - SinglePhaseParallelIndexTaskRunner runner, ParallelIndexIngestionSpec ingestionSpec, Map context, InputSplit inputSplit @@ -625,6 +622,7 @@ public ParallelIndexSubTask newSubTask(int numAttempts) private class TestSubTask extends ParallelIndexSubTask { + private final IndexTaskClientFactory taskClientFactory; private volatile TaskState state = TaskState.RUNNING; TestSubTask( @@ -647,12 +645,7 @@ private class TestSubTask extends ParallelIndexSubTask null, taskClientFactory ); - } - - @Override - public boolean isReady(TaskActionClient taskActionClient) - { - return true; + this.taskClientFactory = taskClientFactory; } @Override @@ -662,30 +655,31 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception Thread.sleep(100); } - final TestFirehose firehose = (TestFirehose) getIngestionSchema().getIOConfig().getFirehoseFactory(); + // build LocalParallelIndexTaskClient + final ParallelIndexTaskClient taskClient = taskClientFactory.build(null, getId(), 0, null, 0); - final List locks = toolbox.getTaskActionClient() - .submit(new SurrogateAction<>(getSupervisorTaskId(), new LockListAction())); - Preconditions.checkState(locks.size() == 1, "There should be a single lock"); + final SegmentAllocator segmentAllocator = createSegmentAllocator(toolbox, taskClient); - task.getRunner().collectReport( - new PushedSegmentsReport( - getId(), - Collections.singletonList( - new DataSegment( - getDataSource(), - Intervals.of("2017/2018"), - locks.get(0).getVersion(), - null, - null, - null, - new NumberedShardSpec(firehose.ids.get(0), NUM_SUB_TASKS), - 0, - 1L - ) - ) - ) + final SegmentIdWithShardSpec segmentIdentifier = segmentAllocator.allocate( + new MapBasedInputRow(DateTimes.of("2017-01-01"), Collections.emptyList(), Collections.emptyMap()), + getId(), + null, + true ); + + final DataSegment segment = new DataSegment( + segmentIdentifier.getDataSource(), + segmentIdentifier.getInterval(), + segmentIdentifier.getVersion(), + null, + null, + null, + segmentIdentifier.getShardSpec(), + 0, + 1L + ); + + taskClient.report(getId(), Collections.emptySet(), Collections.singleton(segment)); return TaskStatus.fromCode(getId(), state); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java index cf7dd372bfdd..c6e127bdc8f7 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java @@ -19,28 +19,36 @@ package org.apache.druid.indexing.common.task.batch.parallel; +import com.google.common.collect.ImmutableList; import org.apache.druid.client.indexing.IndexingServiceClient; import org.apache.druid.data.input.FiniteFirehoseFactory; import org.apache.druid.data.input.InputSplit; import org.apache.druid.data.input.impl.StringInputRowParser; import org.apache.druid.indexer.TaskState; +import org.apache.druid.indexing.common.LockGranularity; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.task.TaskResource; +import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; import org.apache.druid.segment.realtime.firehose.LocalFirehoseFactory; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.PartitionChunk; import org.joda.time.Interval; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import javax.annotation.Nullable; import java.io.File; @@ -53,11 +61,28 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +@RunWith(Parameterized.class) public class ParallelIndexSupervisorTaskTest extends AbstractParallelIndexSupervisorTaskTest { + @Parameterized.Parameters(name = "{0}") + public static Iterable constructorFeeder() + { + return ImmutableList.of( + new Object[]{LockGranularity.TIME_CHUNK}, + new Object[]{LockGranularity.SEGMENT} + ); + } + + private final LockGranularity lockGranularity; private File inputDir; + public ParallelIndexSupervisorTaskTest(LockGranularity lockGranularity) + { + this.lockGranularity = lockGranularity; + } + @Before public void setup() throws IOException { @@ -127,62 +152,80 @@ public void testIsReady() throws Exception } } - private void runTestWithoutIntervalTask() throws Exception + private void runTestTask(@Nullable Interval interval, Granularity segmentGranularity, boolean appendToExisting) + throws Exception { final ParallelIndexSupervisorTask task = newTask( - null, + interval, + segmentGranularity, new ParallelIndexIOConfig( new LocalFirehoseFactory(inputDir, "test_*", null), - false + appendToExisting ) ); actionClient = createActionClient(task); toolbox = createTaskToolbox(task); prepareTaskForLocking(task); + task.addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, lockGranularity == LockGranularity.TIME_CHUNK); Assert.assertTrue(task.isReady(actionClient)); Assert.assertEquals(TaskState.SUCCESS, task.run(toolbox).getStatusCode()); shutdownTask(task); } - @Test - public void testWithoutInterval() throws Exception + private void runTestTask(@Nullable Interval interval, Granularity segmentGranularity) throws Exception + { + runTestTask(interval, segmentGranularity, false); + } + + private void testRunAndOverwrite(@Nullable Interval inputInterval, Granularity secondSegmentGranularity) + throws Exception { // Ingest all data. - runTestWithoutIntervalTask(); + runTestTask(inputInterval, Granularities.DAY); - // Read the segments for one day. - final Interval interval = Intervals.of("2017-12-24/P1D"); - final List oldSegments = - getStorageCoordinator().getUsedSegmentsForInterval("dataSource", interval); - Assert.assertEquals(1, oldSegments.size()); + final Interval interval = inputInterval == null ? Intervals.ETERNITY : inputInterval; + final List allSegments = getStorageCoordinator().getUsedSegmentsForInterval("dataSource", interval); // Reingest the same data. Each segment should get replaced by a segment with a newer version. - runTestWithoutIntervalTask(); + runTestTask(inputInterval, secondSegmentGranularity); // Verify that the segment has been replaced. - final List newSegments = - getStorageCoordinator().getUsedSegmentsForInterval("dataSource", interval); - Assert.assertEquals(1, newSegments.size()); - Assert.assertTrue(oldSegments.get(0).getVersion().compareTo(newSegments.get(0).getVersion()) < 0); + final List newSegments = getStorageCoordinator().getUsedSegmentsForInterval("dataSource", interval); + allSegments.addAll(newSegments); + final VersionedIntervalTimeline timeline = VersionedIntervalTimeline.forSegments(allSegments); + final List visibles = timeline.lookup(interval) + .stream() + .flatMap(holder -> holder.getObject().stream()) + .map(PartitionChunk::getObject) + .collect(Collectors.toList()); + Assert.assertEquals(newSegments, visibles); + } + + @Test + public void testWithoutInterval() throws Exception + { + testRunAndOverwrite(null, Granularities.DAY); } @Test() public void testRunInParallel() throws Exception { - final ParallelIndexSupervisorTask task = newTask( - Intervals.of("2017/2018"), - new ParallelIndexIOConfig( - new LocalFirehoseFactory(inputDir, "test_*", null), - false - ) - ); - actionClient = createActionClient(task); - toolbox = createTaskToolbox(task); + // Ingest all data. + testRunAndOverwrite(Intervals.of("2017/2018"), Granularities.DAY); + } - prepareTaskForLocking(task); - Assert.assertTrue(task.isReady(actionClient)); - Assert.assertEquals(TaskState.SUCCESS, task.run(toolbox).getStatusCode()); + @Test + public void testWithoutIntervalWithDifferentSegmentGranularity() throws Exception + { + testRunAndOverwrite(null, Granularities.MONTH); + } + + @Test() + public void testRunInParallelWithDifferentSegmentGranularity() throws Exception + { + // Ingest all data. + testRunAndOverwrite(Intervals.of("2017/2018"), Granularities.MONTH); } @Test @@ -206,6 +249,7 @@ public boolean isSplittable() toolbox = createTaskToolbox(task); prepareTaskForLocking(task); + task.addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, lockGranularity == LockGranularity.TIME_CHUNK); Assert.assertTrue(task.isReady(actionClient)); Assert.assertEquals(TaskState.SUCCESS, task.run(toolbox).getStatusCode()); } @@ -224,6 +268,7 @@ public void testPublishEmptySegments() throws Exception toolbox = createTaskToolbox(task); prepareTaskForLocking(task); + task.addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, lockGranularity == LockGranularity.TIME_CHUNK); Assert.assertTrue(task.isReady(actionClient)); Assert.assertEquals(TaskState.SUCCESS, task.run(toolbox).getStatusCode()); } @@ -233,6 +278,7 @@ public void testWith1MaxNumSubTasks() throws Exception { final ParallelIndexSupervisorTask task = newTask( Intervals.of("2017/2018"), + Granularities.DAY, new ParallelIndexIOConfig( new LocalFirehoseFactory(inputDir, "test_*", null), false @@ -265,18 +311,45 @@ public void testWith1MaxNumSubTasks() throws Exception toolbox = createTaskToolbox(task); prepareTaskForLocking(task); + task.addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, lockGranularity == LockGranularity.TIME_CHUNK); Assert.assertTrue(task.isReady(actionClient)); Assert.assertEquals(TaskState.SUCCESS, task.run(toolbox).getStatusCode()); Assert.assertNull("Runner must be null if the task was in the sequential mode", task.getRunner()); } + @Test + public void testAppendToExisting() throws Exception + { + final Interval interval = Intervals.of("2017/2018"); + runTestTask(interval, Granularities.DAY, true); + final List oldSegments = getStorageCoordinator().getUsedSegmentsForInterval("dataSource", interval); + + runTestTask(interval, Granularities.DAY, true); + final List newSegments = getStorageCoordinator().getUsedSegmentsForInterval("dataSource", interval); + Assert.assertTrue(newSegments.containsAll(oldSegments)); + final VersionedIntervalTimeline timeline = VersionedIntervalTimeline.forSegments(newSegments); + final List visibles = timeline.lookup(interval) + .stream() + .flatMap(holder -> holder.getObject().stream()) + .map(PartitionChunk::getObject) + .collect(Collectors.toList()); + Assert.assertEquals(newSegments, visibles); + } + + private ParallelIndexSupervisorTask newTask(@Nullable Interval interval, ParallelIndexIOConfig ioConfig) + { + return newTask(interval, Granularities.DAY, ioConfig); + } + private ParallelIndexSupervisorTask newTask( - Interval interval, + @Nullable Interval interval, + Granularity segmentGranularity, ParallelIndexIOConfig ioConfig ) { return newTask( interval, + segmentGranularity, ioConfig, new ParallelIndexTuningConfig( null, @@ -305,7 +378,8 @@ private ParallelIndexSupervisorTask newTask( } private ParallelIndexSupervisorTask newTask( - Interval interval, + @Nullable Interval interval, + Granularity segmentGranularity, ParallelIndexIOConfig ioConfig, ParallelIndexTuningConfig tuningConfig ) @@ -326,7 +400,7 @@ private ParallelIndexSupervisorTask newTask( new LongSumAggregatorFactory("val", "val") }, new UniformGranularitySpec( - Granularities.DAY, + segmentGranularity, Granularities.MINUTE, interval == null ? null : Collections.singletonList(interval) ), @@ -372,6 +446,7 @@ private static class TestSupervisorTask extends TestParallelIndexSupervisorTask @Override ParallelIndexTaskRunner createRunner(TaskToolbox toolbox) { + setToolbox(toolbox); setRunner( new TestRunner( toolbox, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitorTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitorTest.java index 7e86e4072fc9..e5f85cb5dc99 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitorTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitorTest.java @@ -162,7 +162,7 @@ private static class TestTask extends NoopTask TestTask(String id, long runTime, boolean shouldFail) { - super(id, "testDataSource", runTime, 0, null, null, null); + super(id, null, "testDataSource", runTime, 0, null, null, null); this.shouldFail = shouldFail; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java index f17274a5ade2..178611ec1d69 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java @@ -59,6 +59,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.filter.SelectorDimFilter; @@ -101,8 +102,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -116,6 +119,7 @@ public class IngestSegmentFirehoseFactoryTest private static final IndexMergerV9 INDEX_MERGER_V9; private static final IndexIO INDEX_IO; private static final TaskStorage TASK_STORAGE; + private static final IndexerSQLMetadataStorageCoordinator MDC; private static final TaskLockbox TASK_LOCKBOX; private static final Task TASK; @@ -132,7 +136,48 @@ public class IngestSegmentFirehoseFactoryTest { } ); - TASK_LOCKBOX = new TaskLockbox(TASK_STORAGE); + MDC = new IndexerSQLMetadataStorageCoordinator(null, null, null) + { + private final Set published = new HashSet<>(); + + @Override + public List getUsedSegmentsForInterval(String dataSource, Interval interval) + { + return ImmutableList.copyOf(segmentSet); + } + + @Override + public List getUsedSegmentsForIntervals(String dataSource, List interval) + { + return ImmutableList.copyOf(segmentSet); + } + + @Override + public List getUnusedSegmentsForInterval(String dataSource, Interval interval) + { + return ImmutableList.of(); + } + + @Override + public Set announceHistoricalSegments(Set segments) + { + Set added = new HashSet<>(); + for (final DataSegment segment : segments) { + if (published.add(segment)) { + added.add(segment); + } + } + + return ImmutableSet.copyOf(added); + } + + @Override + public void deleteSegments(Set segments) + { + // do nothing + } + }; + TASK_LOCKBOX = new TaskLockbox(TASK_STORAGE, MDC); TASK = NoopTask.create(); TASK_LOCKBOX.add(TASK); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RealtimeishTask.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RealtimeishTask.java index 24291e8257a0..8724ba9c125a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RealtimeishTask.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RealtimeishTask.java @@ -25,11 +25,11 @@ import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskToolbox; -import org.apache.druid.indexing.common.actions.LockAcquireAction; import org.apache.druid.indexing.common.actions.LockListAction; import org.apache.druid.indexing.common.actions.LockReleaseAction; import org.apache.druid.indexing.common.actions.SegmentInsertAction; import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TimeChunkLockAcquireAction; import org.apache.druid.indexing.common.task.AbstractTask; import org.apache.druid.indexing.common.task.TaskResource; import org.apache.druid.java.util.common.Intervals; @@ -70,7 +70,7 @@ public TaskStatus run(TaskToolbox toolbox) throws Exception // Acquire lock for first interval final TaskLock lock1 = toolbox.getTaskActionClient().submit( - new LockAcquireAction(TaskLockType.EXCLUSIVE, interval1, 5000) + new TimeChunkLockAcquireAction(TaskLockType.EXCLUSIVE, interval1, 5000) ); Assert.assertNotNull(lock1); final List locks1 = toolbox.getTaskActionClient().submit(new LockListAction()); @@ -81,7 +81,7 @@ public TaskStatus run(TaskToolbox toolbox) throws Exception // Acquire lock for second interval final TaskLock lock2 = toolbox.getTaskActionClient().submit( - new LockAcquireAction(TaskLockType.EXCLUSIVE, interval2, 5000) + new TimeChunkLockAcquireAction(TaskLockType.EXCLUSIVE, interval2, 5000) ); Assert.assertNotNull(lock2); final List locks2 = toolbox.getTaskActionClient().submit(new LockListAction()); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java index a16a80a244de..077ae47490e5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java @@ -127,7 +127,7 @@ public void testRun() throws ExecutionException, InterruptedException { Assert.assertEquals( TaskState.SUCCESS, - runner.run(new NoopTask(null, null, 500L, 0, null, null, null)).get().getStatusCode() + runner.run(new NoopTask(null, null, null, 500L, 0, null, null, null)).get().getStatusCode() ); } @@ -135,7 +135,7 @@ public void testRun() throws ExecutionException, InterruptedException public void testStop() throws ExecutionException, InterruptedException, TimeoutException { final ListenableFuture future = runner.run( - new NoopTask(null, null, Long.MAX_VALUE, 0, null, null, null) // infinite task + new NoopTask(null, null, null, Long.MAX_VALUE, 0, null, null, null) // infinite task ); runner.stop(); Assert.assertEquals( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java index 8139f8dcae13..b89d8f960a19 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java @@ -46,6 +46,7 @@ import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.SegmentLoaderFactory; import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.TaskToolboxFactory; import org.apache.druid.indexing.common.TestUtils; @@ -55,6 +56,7 @@ import org.apache.druid.indexing.common.actions.TaskActionClientFactory; import org.apache.druid.indexing.common.actions.TaskActionToolbox; import org.apache.druid.indexing.common.actions.TaskAuditLogConfig; +import org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction; import org.apache.druid.indexing.common.config.TaskConfig; import org.apache.druid.indexing.common.config.TaskStorageConfig; import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory; @@ -527,7 +529,7 @@ private TaskToolboxFactory setUpTaskToolboxFactory( Preconditions.checkNotNull(taskStorage); Preconditions.checkNotNull(emitter); - taskLockbox = new TaskLockbox(taskStorage); + taskLockbox = new TaskLockbox(taskStorage, mdc); tac = new LocalTaskActionClientFactory( taskStorage, new TaskActionToolbox( @@ -936,15 +938,21 @@ public String getType() @Override public TaskStatus run(TaskToolbox toolbox) throws Exception { - final TaskLock myLock = Iterables.getOnlyElement( - toolbox.getTaskActionClient() - .submit(new LockListAction()) + final Interval interval = Intervals.of("2012-01-01/P1D"); + final TimeChunkLockTryAcquireAction action = new TimeChunkLockTryAcquireAction( + TaskLockType.EXCLUSIVE, + interval ); + final TaskLock lock = toolbox.getTaskActionClient().submit(action); + if (lock == null) { + throw new ISE("Failed to get a lock"); + } + final DataSegment segment = DataSegment.builder() .dataSource("ds") - .interval(Intervals.of("2012-01-01/P1D")) - .version(myLock.getVersion()) + .interval(interval) + .version(lock.getVersion()) .build(); toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment))); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java index 10b46c88fba6..761dee61efda 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.metadata.DerbyMetadataStorageActionHandlerFactory; import org.apache.druid.metadata.EntryExistsException; +import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator; import org.apache.druid.metadata.TestDerbyConnector; import org.joda.time.Interval; import org.junit.After; @@ -72,7 +73,10 @@ public void setup() ) ); - lockbox = new TaskLockbox(taskStorage); + lockbox = new TaskLockbox( + taskStorage, + new IndexerSQLMetadataStorageCoordinator(objectMapper, derby.metadataTablesConfigSupplier().get(), derbyConnector) + ); service = Executors.newFixedThreadPool(2); } @@ -82,6 +86,17 @@ public void teardown() service.shutdownNow(); } + private LockResult tryTimeChunkLock(TaskLockType lockType, Task task, Interval interval) + { + return lockbox.tryLock(task, new TimeChunkLockRequest(lockType, task, interval, null)); + } + + private LockResult acquireTimeChunkLock(TaskLockType lockType, Task task, Interval interval) + throws InterruptedException + { + return lockbox.lock(task, new TimeChunkLockRequest(lockType, task, interval, null)); + } + @Test(timeout = 60_000L) public void testDoInCriticalSectionWithDifferentTasks() throws ExecutionException, InterruptedException, EntryExistsException @@ -99,7 +114,7 @@ public void testDoInCriticalSectionWithDifferentTasks() // lowPriorityTask acquires a lock first and increases the int of intSupplier in the critical section final Future lowPriorityFuture = service.submit(() -> { - final LockResult result = lockbox.tryLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval); + final LockResult result = tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval); Assert.assertTrue(result.isOk()); Assert.assertFalse(result.isRevoked()); @@ -129,7 +144,7 @@ public void testDoInCriticalSectionWithDifferentTasks() // section final Future highPriorityFuture = service.submit(() -> { latch.await(); - final LockResult result = lockbox.lock(TaskLockType.EXCLUSIVE, highPriorityTask, interval); + final LockResult result = acquireTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval); Assert.assertTrue(result.isOk()); Assert.assertFalse(result.isRevoked()); @@ -158,7 +173,7 @@ public void testDoInCriticalSectionWithDifferentTasks() Assert.assertEquals(2, highPriorityFuture.get().intValue()); // the lock for lowPriorityTask must be revoked by the highPriorityTask after its work is done in critical section - final LockResult result = lockbox.tryLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval); + final LockResult result = tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval); Assert.assertFalse(result.isOk()); Assert.assertTrue(result.isRevoked()); } @@ -176,7 +191,7 @@ public void testDoInCriticalSectionWithOverlappedIntervals() throws Exception taskStorage.insert(task, TaskStatus.running(task.getId())); for (Interval interval : intervals) { - final LockResult result = lockbox.tryLock(TaskLockType.EXCLUSIVE, task, interval); + final LockResult result = tryTimeChunkLock(TaskLockType.EXCLUSIVE, task, interval); Assert.assertTrue(result.isOk()); } @@ -185,7 +200,7 @@ public void testDoInCriticalSectionWithOverlappedIntervals() throws Exception final Future future1 = service.submit(() -> lockbox.doInCriticalSection( task, - ImmutableList.of(intervals.get(0), intervals.get(1)), + intervals.subList(0, 2), CriticalAction.builder() .onValidLocks( () -> { @@ -208,7 +223,7 @@ public void testDoInCriticalSectionWithOverlappedIntervals() throws Exception latch.await(); return lockbox.doInCriticalSection( task, - ImmutableList.of(intervals.get(1), intervals.get(2)), + intervals.subList(1, 3), CriticalAction.builder() .onValidLocks( () -> { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java index a2a14a006bba..004c15b3527e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java @@ -27,9 +27,12 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.Iterables; import org.apache.druid.indexer.TaskStatus; +import org.apache.druid.indexing.common.LockGranularity; +import org.apache.druid.indexing.common.SegmentLock; import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskToolbox; +import org.apache.druid.indexing.common.TimeChunkLock; import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.config.TaskStorageConfig; import org.apache.druid.indexing.common.task.AbstractTask; @@ -37,6 +40,7 @@ import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.TaskLockbox.TaskLockPosse; import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; @@ -44,7 +48,18 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.metadata.DerbyMetadataStorageActionHandlerFactory; import org.apache.druid.metadata.EntryExistsException; +import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator; +import org.apache.druid.metadata.MetadataStorageTablesConfig; import org.apache.druid.metadata.TestDerbyConnector; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; +import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec; +import org.apache.druid.timeline.partition.HashBasedNumberedShardSpecFactory; +import org.apache.druid.timeline.partition.NumberedOverwritingShardSpecFactory; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpecFactory; +import org.apache.druid.timeline.partition.PartitionIds; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.easymock.EasyMock; import org.joda.time.Interval; import org.junit.Assert; @@ -70,8 +85,9 @@ public class TaskLockboxTest @Rule public ExpectedException expectedException = ExpectedException.none(); - private final ObjectMapper objectMapper = new DefaultObjectMapper(); + private ObjectMapper objectMapper; private TaskStorage taskStorage; + private IndexerMetadataStorageCoordinator metadataStorageCoordinator; private TaskLockbox lockbox; @Rule @@ -80,14 +96,20 @@ public class TaskLockboxTest @Before public void setup() { + objectMapper = TestHelper.makeJsonMapper(); + objectMapper.registerSubtypes(NumberedShardSpec.class, HashBasedNumberedShardSpec.class); + final TestDerbyConnector derbyConnector = derby.getConnector(); derbyConnector.createTaskTables(); + derbyConnector.createPendingSegmentsTable(); + derbyConnector.createSegmentTable(); + final MetadataStorageTablesConfig tablesConfig = derby.metadataTablesConfigSupplier().get(); taskStorage = new MetadataTaskStorage( derbyConnector, new TaskStorageConfig(null), new DerbyMetadataStorageActionHandlerFactory( derbyConnector, - derby.metadataTablesConfigSupplier().get(), + tablesConfig, objectMapper ) ); @@ -95,7 +117,26 @@ public void setup() EmittingLogger.registerEmitter(emitter); EasyMock.replay(emitter); - lockbox = new TaskLockbox(taskStorage); + metadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator(objectMapper, tablesConfig, derbyConnector); + + lockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator); + } + + private LockResult acquireTimeChunkLock(TaskLockType lockType, Task task, Interval interval, long timeoutMs) + throws InterruptedException + { + return lockbox.lock(task, new TimeChunkLockRequest(lockType, task, interval, null), timeoutMs); + } + + private LockResult acquireTimeChunkLock(TaskLockType lockType, Task task, Interval interval) + throws InterruptedException + { + return lockbox.lock(task, new TimeChunkLockRequest(lockType, task, interval, null)); + } + + private LockResult tryTimeChunkLock(TaskLockType lockType, Task task, Interval interval) + { + return lockbox.tryLock(task, new TimeChunkLockRequest(lockType, task, interval, null)); } @Test @@ -103,13 +144,13 @@ public void testLock() throws InterruptedException { Task task = NoopTask.create(); lockbox.add(task); - Assert.assertNotNull(lockbox.lock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-02"))); + Assert.assertNotNull(acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-02"))); } @Test(expected = IllegalStateException.class) public void testLockForInactiveTask() throws InterruptedException { - lockbox.lock(TaskLockType.EXCLUSIVE, NoopTask.create(), Intervals.of("2015-01-01/2015-01-02")); + acquireTimeChunkLock(TaskLockType.EXCLUSIVE, NoopTask.create(), Intervals.of("2015-01-01/2015-01-02")); } @Test @@ -120,7 +161,7 @@ public void testLockAfterTaskComplete() throws InterruptedException exception.expectMessage("Unable to grant lock to inactive Task"); lockbox.add(task); lockbox.remove(task); - lockbox.lock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-02")); + acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-02")); } @Test @@ -135,7 +176,7 @@ public void testTrySharedLock() final Task task = NoopTask.create(Math.min(0, (i - 1) * 10)); // the first two tasks have the same priority tasks.add(task); lockbox.add(task); - final TaskLock lock = lockbox.tryLock(TaskLockType.SHARED, task, interval).getTaskLock(); + final TaskLock lock = tryTimeChunkLock(TaskLockType.SHARED, task, interval).getTaskLock(); Assert.assertNotNull(lock); actualLocks.add(lock); } @@ -160,15 +201,15 @@ public void testTryMixedLocks() throws EntryExistsException lockbox.add(lowPriorityTask); lockbox.add(lowPriorityTask2); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval1).isOk()); - Assert.assertTrue(lockbox.tryLock(TaskLockType.SHARED, lowPriorityTask, interval2).isOk()); - Assert.assertTrue(lockbox.tryLock(TaskLockType.SHARED, lowPriorityTask2, interval2).isOk()); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval3).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval1).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, lowPriorityTask, interval2).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, lowPriorityTask2, interval2).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval3).isOk()); lockbox.add(highPiorityTask); - Assert.assertTrue(lockbox.tryLock(TaskLockType.SHARED, highPiorityTask, interval1).isOk()); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval2).isOk()); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval3).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, highPiorityTask, interval1).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval2).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval3).isOk()); Assert.assertTrue(lockbox.findLocksForTask(lowPriorityTask).stream().allMatch(TaskLock::isRevoked)); Assert.assertTrue(lockbox.findLocksForTask(lowPriorityTask2).stream().allMatch(TaskLock::isRevoked)); @@ -178,14 +219,14 @@ public void testTryMixedLocks() throws EntryExistsException lockbox.remove(highPiorityTask); lockbox.add(highPiorityTask); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval1).isOk()); - Assert.assertTrue(lockbox.tryLock(TaskLockType.SHARED, highPiorityTask, interval2).isOk()); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval3).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval1).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, highPiorityTask, interval2).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval3).isOk()); lockbox.add(lowPriorityTask); - Assert.assertFalse(lockbox.tryLock(TaskLockType.SHARED, lowPriorityTask, interval1).isOk()); - Assert.assertFalse(lockbox.tryLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval2).isOk()); - Assert.assertFalse(lockbox.tryLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval3).isOk()); + Assert.assertFalse(tryTimeChunkLock(TaskLockType.SHARED, lowPriorityTask, interval1).isOk()); + Assert.assertFalse(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval2).isOk()); + Assert.assertFalse(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval3).isOk()); } @Test @@ -193,24 +234,24 @@ public void testTryExclusiveLock() { Task task = NoopTask.create(); lockbox.add(task); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-03")).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-03")).isOk()); // try to take lock for task 2 for overlapping interval Task task2 = NoopTask.create(); lockbox.add(task2); - Assert.assertFalse(lockbox.tryLock(TaskLockType.EXCLUSIVE, task2, Intervals.of("2015-01-01/2015-01-02")).isOk()); + Assert.assertFalse(tryTimeChunkLock(TaskLockType.EXCLUSIVE, task2, Intervals.of("2015-01-01/2015-01-02")).isOk()); // task 1 unlocks the lock lockbox.remove(task); // Now task2 should be able to get the lock - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, task2, Intervals.of("2015-01-01/2015-01-02")).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, task2, Intervals.of("2015-01-01/2015-01-02")).isOk()); } @Test(expected = IllegalStateException.class) public void testTryLockForInactiveTask() { - Assert.assertFalse(lockbox.tryLock(TaskLockType.EXCLUSIVE, NoopTask.create(), Intervals.of("2015-01-01/2015-01-02")).isOk()); + Assert.assertFalse(tryTimeChunkLock(TaskLockType.EXCLUSIVE, NoopTask.create(), Intervals.of("2015-01-01/2015-01-02")).isOk()); } @Test @@ -221,7 +262,7 @@ public void testTryLockAfterTaskComplete() exception.expectMessage("Unable to grant lock to inactive Task"); lockbox.add(task); lockbox.remove(task); - Assert.assertFalse(lockbox.tryLock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-02")).isOk()); + Assert.assertFalse(tryTimeChunkLock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-02")).isOk()); } @Test @@ -232,23 +273,27 @@ public void testTimeoutForLock() throws InterruptedException lockbox.add(task1); lockbox.add(task2); - Assert.assertTrue(lockbox.lock(TaskLockType.EXCLUSIVE, task1, Intervals.of("2015-01-01/2015-01-02"), 5000).isOk()); - Assert.assertFalse(lockbox.lock(TaskLockType.EXCLUSIVE, task2, Intervals.of("2015-01-01/2015-01-15"), 1000).isOk()); + Assert.assertTrue(acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task1, Intervals.of("2015-01-01/2015-01-02"), 5000).isOk()); + Assert.assertFalse(acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task2, Intervals.of("2015-01-01/2015-01-15"), 1000).isOk()); } @Test public void testSyncFromStorage() throws EntryExistsException { - final TaskLockbox originalBox = new TaskLockbox(taskStorage); + final TaskLockbox originalBox = new TaskLockbox(taskStorage, metadataStorageCoordinator); for (int i = 0; i < 5; i++) { final Task task = NoopTask.create(); taskStorage.insert(task, TaskStatus.running(task.getId())); originalBox.add(task); Assert.assertTrue( originalBox.tryLock( - TaskLockType.EXCLUSIVE, task, - Intervals.of(StringUtils.format("2017-01-0%d/2017-01-0%d", (i + 1), (i + 2))) + new TimeChunkLockRequest( + TaskLockType.EXCLUSIVE, + task, + Intervals.of(StringUtils.format("2017-01-0%d/2017-01-0%d", (i + 1), (i + 2))), + null + ) ).isOk() ); } @@ -257,7 +302,7 @@ public void testSyncFromStorage() throws EntryExistsException .flatMap(task -> taskStorage.getLocks(task.getId()).stream()) .collect(Collectors.toList()); - final TaskLockbox newBox = new TaskLockbox(taskStorage); + final TaskLockbox newBox = new TaskLockbox(taskStorage, metadataStorageCoordinator); newBox.syncFromStorage(); Assert.assertEquals(originalBox.getAllLocks(), newBox.getAllLocks()); @@ -277,14 +322,14 @@ public void testSyncFromStorageWithMissingTaskLockPriority() throws EntryExistsE taskStorage.insert(task, TaskStatus.running(task.getId())); taskStorage.addLock( task.getId(), - new TaskLockWithoutPriority(task.getGroupId(), task.getDataSource(), Intervals.of("2017/2018"), "v1") + new IntervalLockWithoutPriority(task.getGroupId(), task.getDataSource(), Intervals.of("2017/2018"), "v1") ); final List beforeLocksInStorage = taskStorage.getActiveTasks().stream() .flatMap(t -> taskStorage.getLocks(t.getId()).stream()) .collect(Collectors.toList()); - final TaskLockbox lockbox = new TaskLockbox(taskStorage); + final TaskLockbox lockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator); lockbox.syncFromStorage(); final List afterLocksInStorage = taskStorage.getActiveTasks().stream() @@ -301,7 +346,7 @@ public void testSyncFromStorageWithMissingTaskPriority() throws EntryExistsExcep taskStorage.insert(task, TaskStatus.running(task.getId())); taskStorage.addLock( task.getId(), - new TaskLock( + new TimeChunkLock( TaskLockType.EXCLUSIVE, task.getGroupId(), task.getDataSource(), @@ -315,7 +360,7 @@ public void testSyncFromStorageWithMissingTaskPriority() throws EntryExistsExcep .flatMap(t -> taskStorage.getLocks(t.getId()).stream()) .collect(Collectors.toList()); - final TaskLockbox lockbox = new TaskLockbox(taskStorage); + final TaskLockbox lockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator); lockbox.syncFromStorage(); final List afterLocksInStorage = taskStorage.getActiveTasks().stream() @@ -332,7 +377,7 @@ public void testSyncFromStorageWithInvalidPriority() throws EntryExistsException taskStorage.insert(task, TaskStatus.running(task.getId())); taskStorage.addLock( task.getId(), - new TaskLock( + new TimeChunkLock( TaskLockType.EXCLUSIVE, task.getGroupId(), task.getDataSource(), @@ -342,7 +387,7 @@ public void testSyncFromStorageWithInvalidPriority() throws EntryExistsException ) ); - final TaskLockbox lockbox = new TaskLockbox(taskStorage); + final TaskLockbox lockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("lock priority[10] is different from task priority[50]"); lockbox.syncFromStorage(); @@ -364,9 +409,14 @@ public void testSyncWithUnknownTaskTypesFromModuleNotLoaded() throws Exception loadedMapper ) ); + IndexerMetadataStorageCoordinator loadedMetadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator( + loadedMapper, + derby.metadataTablesConfigSupplier().get(), + derbyConnector + ); - TaskLockbox theBox = new TaskLockbox(taskStorage); - TaskLockbox loadedBox = new TaskLockbox(loadedTaskStorage); + TaskLockbox theBox = new TaskLockbox(taskStorage, metadataStorageCoordinator); + TaskLockbox loadedBox = new TaskLockbox(loadedTaskStorage, loadedMetadataStorageCoordinator); Task aTask = NoopTask.create(); taskStorage.insert(aTask, TaskStatus.running(aTask.getId())); @@ -391,18 +441,18 @@ public void testSyncWithUnknownTaskTypesFromModuleNotLoaded() throws Exception @Test public void testRevokedLockSyncFromStorage() throws EntryExistsException { - final TaskLockbox originalBox = new TaskLockbox(taskStorage); + final TaskLockbox originalBox = new TaskLockbox(taskStorage, metadataStorageCoordinator); final Task task1 = NoopTask.create("task1", 10); taskStorage.insert(task1, TaskStatus.running(task1.getId())); originalBox.add(task1); - Assert.assertTrue(originalBox.tryLock(TaskLockType.EXCLUSIVE, task1, Intervals.of("2017/2018")).isOk()); + Assert.assertTrue(originalBox.tryLock(task1, new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task1, Intervals.of("2017/2018"), null)).isOk()); // task2 revokes task1 final Task task2 = NoopTask.create("task2", 100); taskStorage.insert(task2, TaskStatus.running(task2.getId())); originalBox.add(task2); - Assert.assertTrue(originalBox.tryLock(TaskLockType.EXCLUSIVE, task2, Intervals.of("2017/2018")).isOk()); + Assert.assertTrue(originalBox.tryLock(task2, new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task2, Intervals.of("2017/2018"), null)).isOk()); final Map> beforeLocksInStorage = taskStorage .getActiveTasks() @@ -417,7 +467,7 @@ public void testRevokedLockSyncFromStorage() throws EntryExistsException Assert.assertEquals(1, task2Locks.size()); Assert.assertTrue(task2Locks.get(0).isRevoked()); - final TaskLockbox newBox = new TaskLockbox(taskStorage); + final TaskLockbox newBox = new TaskLockbox(taskStorage, metadataStorageCoordinator); newBox.syncFromStorage(); final Set afterLocksInStorage = taskStorage.getActiveTasks().stream() @@ -436,7 +486,7 @@ public void testDoInCriticalSectionWithSharedLock() throws Exception final Interval interval = Intervals.of("2017-01-01/2017-01-02"); final Task task = NoopTask.create(); lockbox.add(task); - Assert.assertTrue(lockbox.tryLock(TaskLockType.SHARED, task, interval).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, task, interval).isOk()); Assert.assertFalse( lockbox.doInCriticalSection( @@ -453,7 +503,7 @@ public void testDoInCriticalSectionWithExclusiveLock() throws Exception final Interval interval = Intervals.of("2017-01-01/2017-01-02"); final Task task = NoopTask.create(); lockbox.add(task); - final TaskLock lock = lockbox.tryLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock(); + final TaskLock lock = tryTimeChunkLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock(); Assert.assertNotNull(lock); Assert.assertTrue( @@ -472,13 +522,13 @@ public void testDoInCriticalSectionWithSmallerInterval() throws Exception final Interval smallInterval = Intervals.of("2017-01-10/2017-01-11"); final Task task = NoopTask.create(); lockbox.add(task); - final TaskLock lock = lockbox.tryLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock(); + final TaskLock lock = tryTimeChunkLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock(); Assert.assertNotNull(lock); Assert.assertTrue( lockbox.doInCriticalSection( task, - Collections.singletonList(smallInterval), + Collections.singletonList(interval), CriticalAction.builder().onValidLocks(() -> true).onInvalidLocks(() -> false).build() ) ); @@ -492,13 +542,13 @@ public void testPreemptionAndDoInCriticalSection() throws Exception final Task task = NoopTask.create(); lockbox.add(task); taskStorage.insert(task, TaskStatus.running(task.getId())); - Assert.assertTrue(lockbox.tryLock(TaskLockType.SHARED, task, interval).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, task, interval).isOk()); } final Task highPriorityTask = NoopTask.create(100); lockbox.add(highPriorityTask); taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId())); - final TaskLock lock = lockbox.tryLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).getTaskLock(); + final TaskLock lock = tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).getTaskLock(); Assert.assertNotNull(lock); Assert.assertTrue( @@ -521,9 +571,9 @@ public void testDoInCriticalSectionWithRevokedLock() throws Exception taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId())); taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId())); - final TaskLock lowPriorityLock = lockbox.tryLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval).getTaskLock(); + final TaskLock lowPriorityLock = tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval).getTaskLock(); Assert.assertNotNull(lowPriorityLock); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).isOk()); Assert.assertTrue(Iterables.getOnlyElement(lockbox.findLocksForTask(lowPriorityTask)).isRevoked()); Assert.assertFalse( @@ -546,15 +596,15 @@ public void testAcquireLockAfterRevoked() throws EntryExistsException, Interrupt taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId())); taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId())); - final TaskLock lowPriorityLock = lockbox.lock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval).getTaskLock(); + final TaskLock lowPriorityLock = acquireTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval).getTaskLock(); Assert.assertNotNull(lowPriorityLock); - Assert.assertTrue(lockbox.tryLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).isOk()); + Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).isOk()); Assert.assertTrue(Iterables.getOnlyElement(lockbox.findLocksForTask(lowPriorityTask)).isRevoked()); lockbox.unlock(highPriorityTask, interval); // Acquire again - final LockResult lockResult = lockbox.lock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval); + final LockResult lockResult = acquireTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval); Assert.assertFalse(lockResult.isOk()); Assert.assertTrue(lockResult.isRevoked()); Assert.assertTrue(Iterables.getOnlyElement(lockbox.findLocksForTask(lowPriorityTask)).isRevoked()); @@ -572,7 +622,7 @@ public void testUnlock() throws EntryExistsException taskStorage.insert(task, TaskStatus.running(task.getId())); lockbox.add(task); Assert.assertTrue( - lockbox.tryLock( + tryTimeChunkLock( TaskLockType.EXCLUSIVE, task, Intervals.of(StringUtils.format("2017-01-0%d/2017-01-0%d", (i + 1), (i + 2))) @@ -587,7 +637,7 @@ public void testUnlock() throws EntryExistsException taskStorage.insert(task, TaskStatus.running(task.getId())); lockbox.add(task); Assert.assertTrue( - lockbox.tryLock( + tryTimeChunkLock( TaskLockType.EXCLUSIVE, task, Intervals.of(StringUtils.format("2017-01-0%d/2017-01-0%d", (i + 1), (i + 2))) @@ -637,34 +687,333 @@ public void testFindLockPosseAfterRevokeWithDifferentLockIntervals() throws Entr lockbox.add(highPriorityTask); Assert.assertTrue( - lockbox.tryLock( + tryTimeChunkLock( TaskLockType.EXCLUSIVE, - lowPriorityTask, Intervals.of("2018-12-16T09:00:00/2018-12-16T10:00:00") + lowPriorityTask, + Intervals.of("2018-12-16T09:00:00/2018-12-16T10:00:00") ).isOk() ); Assert.assertTrue( - lockbox.tryLock( + tryTimeChunkLock( TaskLockType.EXCLUSIVE, - highPriorityTask, Intervals.of("2018-12-16T09:00:00/2018-12-16T09:30:00") + highPriorityTask, + Intervals.of("2018-12-16T09:00:00/2018-12-16T09:30:00") ).isOk() ); - final TaskLockPosse highLockPosse = lockbox.getOnlyTaskLockPosseContainingInterval( + final List highLockPosses = lockbox.getOnlyTaskLockPosseContainingInterval( highPriorityTask, Intervals.of("2018-12-16T09:00:00/2018-12-16T09:30:00") ); - Assert.assertTrue(highLockPosse.containsTask(highPriorityTask)); - Assert.assertFalse(highLockPosse.getTaskLock().isRevoked()); + Assert.assertEquals(1, highLockPosses.size()); + Assert.assertTrue(highLockPosses.get(0).containsTask(highPriorityTask)); + Assert.assertFalse(highLockPosses.get(0).getTaskLock().isRevoked()); - final TaskLockPosse lowLockPosse = lockbox.getOnlyTaskLockPosseContainingInterval( + final List lowLockPosses = lockbox.getOnlyTaskLockPosseContainingInterval( lowPriorityTask, Intervals.of("2018-12-16T09:00:00/2018-12-16T10:00:00") ); - Assert.assertTrue(lowLockPosse.containsTask(lowPriorityTask)); - Assert.assertTrue(lowLockPosse.getTaskLock().isRevoked()); + Assert.assertEquals(1, lowLockPosses.size()); + Assert.assertTrue(lowLockPosses.get(0).containsTask(lowPriorityTask)); + Assert.assertTrue(lowLockPosses.get(0).getTaskLock().isRevoked()); + } + + @Test + public void testSegmentLock() throws InterruptedException + { + final Task task = NoopTask.create(); + lockbox.add(task); + final LockResult lockResult = lockbox.lock( + task, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task, + Intervals.of("2015-01-01/2015-01-02"), + "v1", + 3 + ) + ); + Assert.assertTrue(lockResult.isOk()); + Assert.assertNull(lockResult.getNewSegmentId()); + Assert.assertTrue(lockResult.getTaskLock() instanceof SegmentLock); + final SegmentLock segmentLock = (SegmentLock) lockResult.getTaskLock(); + Assert.assertEquals(TaskLockType.EXCLUSIVE, segmentLock.getType()); + Assert.assertEquals(task.getGroupId(), segmentLock.getGroupId()); + Assert.assertEquals(task.getDataSource(), segmentLock.getDataSource()); + Assert.assertEquals(Intervals.of("2015-01-01/2015-01-02"), segmentLock.getInterval()); + Assert.assertEquals("v1", segmentLock.getVersion()); + Assert.assertEquals(3, segmentLock.getPartitionId()); + Assert.assertEquals(task.getPriority(), segmentLock.getPriority().intValue()); + Assert.assertFalse(segmentLock.isRevoked()); + } + + @Test + public void testSegmentAndTimeChunkLockForSameInterval() + { + final Task task1 = NoopTask.create(); + lockbox.add(task1); + + final Task task2 = NoopTask.create(); + lockbox.add(task2); + + Assert.assertTrue( + lockbox.tryLock( + task1, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task1, + Intervals.of("2015-01-01/2015-01-02"), + "v1", + 3 + ) + ).isOk() + ); + + Assert.assertFalse( + lockbox.tryLock( + task2, + new TimeChunkLockRequest( + TaskLockType.EXCLUSIVE, + task2, + Intervals.of("2015-01-01/2015-01-02"), + "v1" + ) + ).isOk() + ); + } + + @Test + public void testSegmentAndTimeChunkLockForSameIntervalWithDifferentPriority() throws EntryExistsException + { + final Task task1 = NoopTask.create(10); + lockbox.add(task1); + taskStorage.insert(task1, TaskStatus.running(task1.getId())); + + final Task task2 = NoopTask.create(100); + lockbox.add(task2); + taskStorage.insert(task2, TaskStatus.running(task2.getId())); + + Assert.assertTrue( + lockbox.tryLock( + task1, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task1, + Intervals.of("2015-01-01/2015-01-02"), + "v1", + 3 + ) + ).isOk() + ); + + Assert.assertTrue( + lockbox.tryLock( + task2, + new TimeChunkLockRequest( + TaskLockType.EXCLUSIVE, + task2, + Intervals.of("2015-01-01/2015-01-02"), + "v1" + ) + ).isOk() + ); + + final LockResult resultOfTask1 = lockbox.tryLock( + task1, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task1, + Intervals.of("2015-01-01/2015-01-02"), + "v1", + 3 + ) + ); + Assert.assertFalse(resultOfTask1.isOk()); + Assert.assertTrue(resultOfTask1.isRevoked()); + } + + @Test + public void testSegmentLockForSameIntervalAndSamePartition() + { + final Task task1 = NoopTask.create(); + lockbox.add(task1); + + final Task task2 = NoopTask.create(); + lockbox.add(task2); + + Assert.assertTrue( + lockbox.tryLock( + task1, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task1, + Intervals.of("2015-01-01/2015-01-02"), + "v1", + 3 + ) + ).isOk() + ); + + Assert.assertFalse( + lockbox.tryLock( + task2, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task2, + Intervals.of("2015-01-01/2015-01-02"), + "v1", + 3 + ) + ).isOk() + ); + } + + @Test + public void testSegmentLockForSameIntervalDifferentPartition() + { + final Task task1 = NoopTask.create(); + lockbox.add(task1); + + final Task task2 = NoopTask.create(); + lockbox.add(task2); + + Assert.assertTrue( + lockbox.tryLock( + task1, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task1, + Intervals.of("2015-01-01/2015-01-02"), + "v1", + 3 + ) + ).isOk() + ); + + Assert.assertTrue( + lockbox.tryLock( + task2, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task2, + Intervals.of("2015-01-01/2015-01-02"), + "v1", + 2 + ) + ).isOk() + ); + } + + @Test + public void testSegmentLockForOverlappedIntervalDifferentPartition() + { + final Task task1 = NoopTask.create(); + lockbox.add(task1); + + final Task task2 = NoopTask.create(); + lockbox.add(task2); + + Assert.assertTrue( + lockbox.tryLock( + task1, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task1, + Intervals.of("2015-01-01/2015-01-05"), + "v1", + 3 + ) + ).isOk() + ); + + Assert.assertFalse( + lockbox.tryLock( + task2, + new SpecificSegmentLockRequest( + TaskLockType.EXCLUSIVE, + task2, + Intervals.of("2015-01-03/2015-01-08"), + "v1", + 2 + ) + ).isOk() + ); + } + + @Test + public void testRequestForNewSegmentWithSegmentLock() + { + final Task task = NoopTask.create(); + lockbox.add(task); + allocateSegmentsAndAssert(task, "seq", 3, NumberedShardSpecFactory.instance()); + allocateSegmentsAndAssert(task, "seq2", 2, new NumberedOverwritingShardSpecFactory(0, 3, (short) 1)); + + final List locks = lockbox.findLocksForTask(task); + Assert.assertEquals(5, locks.size()); + int expectedPartitionId = 0; + for (TaskLock lock : locks) { + Assert.assertTrue(lock instanceof SegmentLock); + final SegmentLock segmentLock = (SegmentLock) lock; + Assert.assertEquals(expectedPartitionId++, segmentLock.getPartitionId()); + if (expectedPartitionId == 3) { + expectedPartitionId = PartitionIds.NON_ROOT_GEN_START_PARTITION_ID; + } + } + } + + @Test + public void testRequestForNewSegmentWithHashPartition() + { + final Task task = NoopTask.create(); + lockbox.add(task); + + allocateSegmentsAndAssert(task, "seq", 3, new HashBasedNumberedShardSpecFactory(null, 3)); + allocateSegmentsAndAssert(task, "seq2", 5, new HashBasedNumberedShardSpecFactory(null, 5)); + } + + private void allocateSegmentsAndAssert( + Task task, + String baseSequenceName, + int numSegmentsToAllocate, + ShardSpecFactory shardSpecFactory + ) + { + for (int i = 0; i < numSegmentsToAllocate; i++) { + final LockRequestForNewSegment request = new LockRequestForNewSegment( + LockGranularity.SEGMENT, + TaskLockType.EXCLUSIVE, + task, + Intervals.of("2015-01-01/2015-01-05"), + shardSpecFactory, + StringUtils.format("%s_%d", baseSequenceName, i), + null, + true + ); + assertAllocatedSegments(request, lockbox.tryLock(task, request)); + } + } + + private void assertAllocatedSegments( + LockRequestForNewSegment lockRequest, + LockResult result + ) + { + Assert.assertTrue(result.isOk()); + Assert.assertNotNull(result.getTaskLock()); + Assert.assertTrue(result.getTaskLock() instanceof SegmentLock); + Assert.assertNotNull(result.getNewSegmentId()); + final SegmentLock segmentLock = (SegmentLock) result.getTaskLock(); + final SegmentIdWithShardSpec segmentId = result.getNewSegmentId(); + + Assert.assertEquals(lockRequest.getType(), segmentLock.getType()); + Assert.assertEquals(lockRequest.getGroupId(), segmentLock.getGroupId()); + Assert.assertEquals(lockRequest.getDataSource(), segmentLock.getDataSource()); + Assert.assertEquals(lockRequest.getInterval(), segmentLock.getInterval()); + Assert.assertEquals(lockRequest.getShardSpecFactory().getShardSpecClass(), segmentId.getShardSpec().getClass()); + Assert.assertEquals(lockRequest.getPriority(), lockRequest.getPriority()); } @Test @@ -673,19 +1022,23 @@ public void testLockPosseEquals() final Task task1 = NoopTask.create(); final Task task2 = NoopTask.create(); - TaskLock taskLock1 = new TaskLock(TaskLockType.EXCLUSIVE, + TaskLock taskLock1 = new TimeChunkLock( + TaskLockType.EXCLUSIVE, task1.getGroupId(), task1.getDataSource(), Intervals.of("2018/2019"), "v1", - task1.getPriority()); + task1.getPriority() + ); - TaskLock taskLock2 = new TaskLock(TaskLockType.EXCLUSIVE, + TaskLock taskLock2 = new TimeChunkLock( + TaskLockType.EXCLUSIVE, task2.getGroupId(), task2.getDataSource(), Intervals.of("2018/2019"), "v2", - task2.getPriority()); + task2.getPriority() + ); TaskLockPosse taskLockPosse1 = new TaskLockPosse(taskLock1); TaskLockPosse taskLockPosse2 = new TaskLockPosse(taskLock2); @@ -697,6 +1050,74 @@ public void testLockPosseEquals() Assert.assertEquals(taskLockPosse1, taskLockPosse3); } + @Test + public void testGetTimeChunkAndSegmentLockForSameGroup() + { + final Task task1 = NoopTask.withGroupId("groupId"); + final Task task2 = NoopTask.withGroupId("groupId"); + + lockbox.add(task1); + lockbox.add(task2); + + Assert.assertTrue( + lockbox.tryLock( + task1, + new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task1, Intervals.of("2017/2018"), null) + ).isOk() + ); + + Assert.assertTrue( + lockbox.tryLock( + task2, + new SpecificSegmentLockRequest(TaskLockType.EXCLUSIVE, task2, Intervals.of("2017/2018"), "version", 0) + ).isOk() + ); + + final List posses = lockbox + .getAllLocks() + .get(task1.getDataSource()) + .get(DateTimes.of("2017")) + .get(Intervals.of("2017/2018")); + Assert.assertEquals(2, posses.size()); + + Assert.assertEquals(LockGranularity.TIME_CHUNK, posses.get(0).getTaskLock().getGranularity()); + final TimeChunkLock timeChunkLock = (TimeChunkLock) posses.get(0).getTaskLock(); + Assert.assertEquals("none", timeChunkLock.getDataSource()); + Assert.assertEquals("groupId", timeChunkLock.getGroupId()); + Assert.assertEquals(Intervals.of("2017/2018"), timeChunkLock.getInterval()); + + Assert.assertEquals(LockGranularity.SEGMENT, posses.get(1).getTaskLock().getGranularity()); + final SegmentLock segmentLock = (SegmentLock) posses.get(1).getTaskLock(); + Assert.assertEquals("none", segmentLock.getDataSource()); + Assert.assertEquals("groupId", segmentLock.getGroupId()); + Assert.assertEquals(Intervals.of("2017/2018"), segmentLock.getInterval()); + Assert.assertEquals(0, segmentLock.getPartitionId()); + } + + @Test + public void testGetTimeChunkAndSegmentLockForDifferentGroup() + { + final Task task1 = NoopTask.withGroupId("groupId"); + final Task task2 = NoopTask.withGroupId("groupId2"); + + lockbox.add(task1); + lockbox.add(task2); + + Assert.assertTrue( + lockbox.tryLock( + task1, + new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task1, Intervals.of("2017/2018"), null) + ).isOk() + ); + + Assert.assertFalse( + lockbox.tryLock( + task2, + new SpecificSegmentLockRequest(TaskLockType.EXCLUSIVE, task2, Intervals.of("2017/2018"), "version", 0) + ).isOk() + ); + } + private Set getAllLocks(List tasks) { return tasks.stream() @@ -704,17 +1125,17 @@ private Set getAllLocks(List tasks) .collect(Collectors.toSet()); } - private static class TaskLockWithoutPriority extends TaskLock + private static class IntervalLockWithoutPriority extends TimeChunkLock { @JsonCreator - TaskLockWithoutPriority( + IntervalLockWithoutPriority( String groupId, String dataSource, Interval interval, String version ) { - super(null, groupId, dataSource, interval, version, 0, false); + super(null, groupId, dataSource, interval, version, null, false); } @Override diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java index 2503014c2041..58f93a21d7fa 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java @@ -229,7 +229,7 @@ public void testOverlordRun() throws Exception Assert.assertEquals(druidNode.getHostAndPort(), response.getEntity()); final String taskId_0 = "0"; - NoopTask task_0 = new NoopTask(taskId_0, null, 0, 0, null, null, null); + NoopTask task_0 = NoopTask.create(taskId_0, 0); response = overlordResource.taskPost(task_0, req); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("task", taskId_0), response.getEntity()); @@ -262,7 +262,7 @@ public void testOverlordRun() throws Exception // Manually insert task in taskStorage // Verifies sync from storage final String taskId_1 = "1"; - NoopTask task_1 = new NoopTask(taskId_1, null, 0, 0, null, null, null); + NoopTask task_1 = NoopTask.create(taskId_1, 0); taskStorage.insert(task_1, TaskStatus.running(taskId_1)); // Wait for task runner to run task_1 runTaskCountDownLatches[Integer.parseInt(taskId_1)].await(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java index 16c8afea1851..7977164b1ca6 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java @@ -78,7 +78,7 @@ public static Collection data() private final String requestMethod; private final ResourceFilter resourceFilter; private final Injector injector; - private final Task noopTask = new NoopTask(null, null, 0, 0, null, null, null); + private final Task noopTask = NoopTask.create(); private static boolean mockedOnceTsqa; private static boolean mockedOnceSM; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java index 29294b8ca20c..3c0bddbce950 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java @@ -42,7 +42,7 @@ public void testFindWorkerForTask() new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost1", "localhost2", "localhost3")), false) ); - NoopTask noopTask = new NoopTask(null, null, 1, 0, null, null, null) + NoopTask noopTask = new NoopTask(null, null, null, 1, 0, null, null, null) { @Override public String getDataSource() @@ -112,7 +112,7 @@ public void testFindWorkerForTaskWithNulls() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) ); Assert.assertEquals("lhost", worker.getWorker().getHost()); } @@ -135,7 +135,7 @@ public void testIsolation() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) ); Assert.assertNull(worker); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java index 8d8e494f598d..3602d5b57045 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java @@ -88,7 +88,7 @@ public void testFindWorkerForTask() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) { @Override public String getDataSource() @@ -123,7 +123,7 @@ public void testFindWorkerForTaskWhenSameCurrCapacityUsed() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) { @Override public String getDataSource() @@ -159,7 +159,7 @@ public void testOneDisableWorkerDifferentUsedCapacity() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) { @Override public String getDataSource() @@ -195,7 +195,7 @@ public void testOneDisableWorkerSameUsedCapacity() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) { @Override public String getDataSource() @@ -281,7 +281,7 @@ public void testStrongAffinity() private static NoopTask createDummyTask(final String dataSource) { - return new NoopTask(null, null, 1, 0, null, null, null) + return new NoopTask(null, null, null, 1, 0, null, null, null) { @Override public String getDataSource() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java index 73599f502ed6..dc2cba48ad95 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java @@ -58,7 +58,7 @@ public void testFindWorkerForTask() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) { @Override public String getDataSource() @@ -95,7 +95,7 @@ public void testFindWorkerForTaskWithNulls() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) ); Assert.assertEquals("lhost", worker.getWorker().getHost()); } @@ -118,7 +118,7 @@ public void testIsolation() DateTimes.nowUtc() ) ), - new NoopTask(null, null, 1, 0, null, null, null) + new NoopTask(null, null, null, 1, 0, null, null, null) ); Assert.assertNull(worker); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java b/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java index 0eeecd5375bd..1013f2210fb0 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java @@ -19,15 +19,18 @@ package org.apache.druid.indexing.test; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.druid.indexing.overlord.DataSourceMetadata; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.SegmentPublishResult; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.joda.time.Interval; import java.util.ArrayList; @@ -37,6 +40,7 @@ public class TestIndexerMetadataStorageCoordinator implements IndexerMetadataStorageCoordinator { + private final ObjectMapper objectMapper = new DefaultObjectMapper(); private final Set published = Sets.newConcurrentHashSet(); private final Set nuked = Sets.newConcurrentHashSet(); private final List unusedSegments; @@ -125,11 +129,17 @@ public SegmentIdWithShardSpec allocatePendingSegment( String sequenceName, String previousSegmentId, Interval interval, + ShardSpecFactory shardSpecFactory, String maxVersion, boolean skipSegmentLineageCheck ) { - throw new UnsupportedOperationException(); + return new SegmentIdWithShardSpec( + dataSource, + interval, + maxVersion, + shardSpecFactory.create(objectMapper, 0) + ); } @Override diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskManagerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskManagerTest.java index 635bf4ad6b52..b4c21b7e0df8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskManagerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskManagerTest.java @@ -259,6 +259,6 @@ public void testTaskRun() throws Exception private NoopTask createNoopTask(String id) { - return new NoopTask(id, null, 100, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, 0)); + return new NoopTask(id, null, null, 100, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, 0)); } } diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/CoordinatorResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/CoordinatorResourceTestClient.java index 9aafd6548d0a..074c77b11dbf 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/clients/CoordinatorResourceTestClient.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/CoordinatorResourceTestClient.java @@ -44,8 +44,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; public class CoordinatorResourceTestClient { @@ -134,13 +132,12 @@ public List getSegmentIntervals(final String dataSource) } // return a set of the segment versions for the specified datasource - public Set getSegmentVersions(final String dataSource) + public List getAvailableSegments(final String dataSource) { - ArrayList segments; try { StatusResponseHolder response = makeRequest(HttpMethod.GET, getFullSegmentsURL(dataSource)); - segments = jsonMapper.readValue( + return jsonMapper.readValue( response.getContent(), new TypeReference>() { } @@ -149,7 +146,6 @@ public Set getSegmentVersions(final String dataSource) catch (Exception e) { throw new RuntimeException(e); } - return segments.stream().map(s -> s.getVersion()).collect(Collectors.toSet()); } private Map getLoadStatus() diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractITBatchIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractITBatchIndexTest.java index 0c6a64eb0aa6..dd2945887b5e 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractITBatchIndexTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractITBatchIndexTest.java @@ -22,18 +22,21 @@ import com.google.inject.Inject; import org.apache.commons.io.IOUtils; import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.clients.ClientInfoResourceTestClient; import org.apache.druid.testing.utils.RetryUtil; import org.apache.druid.testing.utils.SqlTestQueryHelper; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.TimelineObjectHolder; +import org.apache.druid.timeline.VersionedIntervalTimeline; import org.junit.Assert; import java.io.IOException; import java.io.InputStream; import java.util.List; -import java.util.Set; public class AbstractITBatchIndexTest extends AbstractIndexerTest { @@ -165,7 +168,7 @@ void doIndexTestSqlTest( private void submitTaskAndWait(String taskSpec, String dataSourceName, boolean waitForNewVersion) { - final Set oldVersions = waitForNewVersion ? coordinator.getSegmentVersions(dataSourceName) : null; + final List oldVersions = waitForNewVersion ? coordinator.getAvailableSegments(dataSourceName) : null; long startSubTaskCount = -1; final boolean assertRunsSubTasks = taskSpec.contains("index_parallel"); @@ -193,7 +196,19 @@ private void submitTaskAndWait(String taskSpec, String dataSourceName, boolean w // original segments have loaded. if (waitForNewVersion) { RetryUtil.retryUntilTrue( - () -> !oldVersions.containsAll(coordinator.getSegmentVersions(dataSourceName)), "See a new version" + () -> { + final VersionedIntervalTimeline timeline = VersionedIntervalTimeline.forSegments( + coordinator.getAvailableSegments(dataSourceName) + ); + + final List> holders = timeline.lookup(Intervals.ETERNITY); + return holders + .stream() + .flatMap(holder -> holder.getObject().stream()) + .anyMatch(chunk -> oldVersions.stream() + .anyMatch(oldSegment -> chunk.getObject().overshadows(oldSegment))); + }, + "See a new version" ); } diff --git a/processing/src/main/java/org/apache/druid/segment/ReferenceCountingSegment.java b/processing/src/main/java/org/apache/druid/segment/ReferenceCountingSegment.java index ef98b7515b2e..335cd1ea8c19 100644 --- a/processing/src/main/java/org/apache/druid/segment/ReferenceCountingSegment.java +++ b/processing/src/main/java/org/apache/druid/segment/ReferenceCountingSegment.java @@ -19,7 +19,9 @@ package org.apache.druid.segment; +import com.google.common.base.Preconditions; import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.timeline.Overshadowable; import org.apache.druid.timeline.SegmentId; import org.joda.time.Interval; @@ -33,11 +35,15 @@ * until that. So ReferenceCountingSegment implements something like automatic reference count-based resource * management. */ -public class ReferenceCountingSegment extends AbstractSegment +public class ReferenceCountingSegment extends AbstractSegment implements Overshadowable { private static final EmittingLogger log = new EmittingLogger(ReferenceCountingSegment.class); private final Segment baseSegment; + private final short startRootPartitionId; + private final short endRootPartitionId; + private final short minorVersion; + private final short atomicUpdateGroupSize; private final AtomicBoolean closed = new AtomicBoolean(false); private final Phaser referents = new Phaser(1) { @@ -65,8 +71,29 @@ protected boolean onAdvance(int phase, int registeredParties) }; public ReferenceCountingSegment(Segment baseSegment) + { + this( + Preconditions.checkNotNull(baseSegment, "baseSegment"), + baseSegment.getId().getPartitionNum(), + (baseSegment.getId().getPartitionNum() + 1), + (short) 0, + (short) 1 + ); + } + + public ReferenceCountingSegment( + Segment baseSegment, + int startRootPartitionId, + int endRootPartitionId, + short minorVersion, + short atomicUpdateGroupSize + ) { this.baseSegment = baseSegment; + this.startRootPartitionId = (short) startRootPartitionId; + this.endRootPartitionId = (short) endRootPartitionId; + this.minorVersion = minorVersion; + this.atomicUpdateGroupSize = atomicUpdateGroupSize; } public Segment getBaseSegment() @@ -150,4 +177,56 @@ public T as(Class clazz) { return getBaseSegment().as(clazz); } + + @Override + public boolean overshadows(ReferenceCountingSegment other) + { + if (baseSegment.getId().getDataSource().equals(other.baseSegment.getId().getDataSource()) + && baseSegment.getId().getInterval().overlaps(other.baseSegment.getId().getInterval())) { + final int majorVersionCompare = baseSegment.getId().getVersion() + .compareTo(other.baseSegment.getId().getVersion()); + if (majorVersionCompare > 0) { + return true; + } else if (majorVersionCompare == 0) { + return includeRootPartitions(other) && getMinorVersion() > other.getMinorVersion(); + } + } + return false; + } + + private boolean includeRootPartitions(ReferenceCountingSegment other) + { + return startRootPartitionId <= other.startRootPartitionId + && endRootPartitionId >= other.endRootPartitionId; + } + + @Override + public int getStartRootPartitionId() + { + return startRootPartitionId; + } + + @Override + public int getEndRootPartitionId() + { + return endRootPartitionId; + } + + @Override + public String getVersion() + { + return baseSegment.getId().getVersion(); + } + + @Override + public short getMinorVersion() + { + return minorVersion; + } + + @Override + public short getAtomicUpdateGroupSize() + { + return atomicUpdateGroupSize; + } } diff --git a/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java index 944139e9cd5d..78b79c0d2157 100644 --- a/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java @@ -57,6 +57,7 @@ import org.apache.druid.segment.IncrementalIndexSegment; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.QueryableIndexSegment; +import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.incremental.IncrementalIndex; @@ -444,7 +445,7 @@ public String toString() } public static QueryRunner makeFilteringQueryRunner( - final VersionedIntervalTimeline timeline, + final VersionedIntervalTimeline timeline, final QueryRunnerFactory> factory ) { @@ -462,7 +463,7 @@ public Sequence run(QueryPlus queryPlus, Map responseConte segments.addAll(timeline.lookup(interval)); } List> sequences = new ArrayList<>(); - for (TimelineObjectHolder holder : toolChest.filterSegments(query, segments)) { + for (TimelineObjectHolder holder : toolChest.filterSegments(query, segments)) { Segment segment = holder.getObject().getChunk(0).getObject(); QueryPlus queryPlusRunning = queryPlus.withQuerySegmentSpec( new SpecificSegmentSpec( diff --git a/processing/src/test/java/org/apache/druid/query/select/MultiSegmentSelectQueryTest.java b/processing/src/test/java/org/apache/druid/query/select/MultiSegmentSelectQueryTest.java index d338b9e4a40d..3ca68a0830ba 100644 --- a/processing/src/test/java/org/apache/druid/query/select/MultiSegmentSelectQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/select/MultiSegmentSelectQueryTest.java @@ -42,6 +42,7 @@ import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.ordering.StringComparators; import org.apache.druid.segment.IncrementalIndexSegment; +import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.incremental.IncrementalIndex; @@ -147,11 +148,10 @@ public static void setup() throws IOException segment1 = new IncrementalIndexSegment(index1, makeIdentifier(index1, "v1")); segment_override = new IncrementalIndexSegment(index2, makeIdentifier(index2, "v2")); - VersionedIntervalTimeline timeline = - new VersionedIntervalTimeline<>(StringComparators.LEXICOGRAPHIC); - timeline.add(index0.getInterval(), "v1", new SingleElementPartitionChunk<>(segment0)); - timeline.add(index1.getInterval(), "v1", new SingleElementPartitionChunk<>(segment1)); - timeline.add(index2.getInterval(), "v2", new SingleElementPartitionChunk<>(segment_override)); + VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(StringComparators.LEXICOGRAPHIC); + timeline.add(index0.getInterval(), "v1", new SingleElementPartitionChunk<>(new ReferenceCountingSegment(segment0))); + timeline.add(index1.getInterval(), "v1", new SingleElementPartitionChunk<>(new ReferenceCountingSegment(segment1))); + timeline.add(index2.getInterval(), "v2", new SingleElementPartitionChunk<>(new ReferenceCountingSegment(segment_override))); segmentIdentifiers = new ArrayList<>(); for (TimelineObjectHolder holder : timeline.lookup(Intervals.of("2011-01-12/2011-01-14"))) { diff --git a/processing/src/test/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java index 769d5cf0e3e8..e999090fb9f8 100644 --- a/processing/src/test/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java @@ -34,6 +34,7 @@ import org.apache.druid.query.TableDataSource; import org.apache.druid.query.ordering.StringComparators; import org.apache.druid.segment.IncrementalIndexSegment; +import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.incremental.IncrementalIndex; @@ -144,10 +145,9 @@ private QueryRunner getCustomRunner() throws IOException segment0 = new IncrementalIndexSegment(index0, makeIdentifier(index0, "v1")); segment1 = new IncrementalIndexSegment(index1, makeIdentifier(index1, "v1")); - VersionedIntervalTimeline timeline = - new VersionedIntervalTimeline<>(StringComparators.LEXICOGRAPHIC); - timeline.add(index0.getInterval(), "v1", new SingleElementPartitionChunk<>(segment0)); - timeline.add(index1.getInterval(), "v1", new SingleElementPartitionChunk<>(segment1)); + VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(StringComparators.LEXICOGRAPHIC); + timeline.add(index0.getInterval(), "v1", new SingleElementPartitionChunk<>(new ReferenceCountingSegment(segment0))); + timeline.add(index1.getInterval(), "v1", new SingleElementPartitionChunk<>(new ReferenceCountingSegment(segment1))); return QueryRunnerTestHelper.makeFilteringQueryRunner(timeline, factory); } diff --git a/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java b/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java index 277367a0254f..43a9ba59c2a5 100644 --- a/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java @@ -41,6 +41,7 @@ import org.apache.druid.segment.incremental.IndexSizeExceededException; import org.apache.druid.segment.serde.ComplexMetrics; import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; +import org.apache.druid.timeline.Overshadowable; import org.apache.druid.timeline.TimelineObjectHolder; import org.apache.druid.timeline.VersionedIntervalTimeline; import org.apache.druid.timeline.partition.NoneShardSpec; @@ -472,14 +473,14 @@ private QueryableIndex makeAppendedMMappedIndex( List filesToMap = makeFilesToMap(tmpFile, files); - VersionedIntervalTimeline timeline = new VersionedIntervalTimeline( + VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>( Comparators.naturalNullsFirst() ); ShardSpec noneShardSpec = NoneShardSpec.instance(); for (int i = 0; i < intervals.size(); i++) { - timeline.add(intervals.get(i), i, noneShardSpec.createChunk(filesToMap.get(i))); + timeline.add(intervals.get(i), i, noneShardSpec.createChunk(new OvershadowableFile(i, filesToMap.get(i)))); } final List adapters = Lists.newArrayList( @@ -487,23 +488,23 @@ private QueryableIndex makeAppendedMMappedIndex( // TimelineObjectHolder is actually an iterable of iterable of indexable adapters Iterables.transform( timeline.lookup(Intervals.of("1000-01-01/3000-01-01")), - new Function, Iterable>() + new Function, Iterable>() { @Override - public Iterable apply(final TimelineObjectHolder timelineObjectHolder) + public Iterable apply(final TimelineObjectHolder timelineObjectHolder) { return Iterables.transform( timelineObjectHolder.getObject(), // Each chunk can be used to build the actual IndexableAdapter - new Function, IndexableAdapter>() + new Function, IndexableAdapter>() { @Override - public IndexableAdapter apply(PartitionChunk chunk) + public IndexableAdapter apply(PartitionChunk chunk) { try { return new RowFilteringIndexAdapter( - new QueryableIndexIndexableAdapter(indexIO.loadIndex(chunk.getObject())), + new QueryableIndexIndexableAdapter(indexIO.loadIndex(chunk.getObject().file)), rowPointer -> timelineObjectHolder.getInterval().contains(rowPointer.getTimestamp()) ); } @@ -569,4 +570,52 @@ public QueryableIndex apply(@Nullable File input) throw new RuntimeException(e); } } + + private static class OvershadowableFile implements Overshadowable + { + private final String majorVersion; + private final File file; + + OvershadowableFile(int majorVersion, File file) + { + this.majorVersion = Integer.toString(majorVersion); + this.file = file; + } + + @Override + public boolean overshadows(OvershadowableFile other) + { + return false; + } + + @Override + public int getStartRootPartitionId() + { + return 0; + } + + @Override + public int getEndRootPartitionId() + { + return 0; + } + + @Override + public String getVersion() + { + return majorVersion; + } + + @Override + public short getMinorVersion() + { + return 0; + } + + @Override + public short getAtomicUpdateGroupSize() + { + return 0; + } + } } diff --git a/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java b/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java index 865b8bd0363a..dbd3ecb35124 100644 --- a/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java +++ b/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java @@ -146,7 +146,7 @@ public Iterable iterateAllUsedSegmentsInSnapshot() /** * This method builds timelines from all data sources and finds the overshadowed segments list * - * This method should be deduplicated with {@link VersionedIntervalTimeline#findOvershadowed()}: see + * This method should be deduplicated with {@link VersionedIntervalTimeline#findFullyOvershadowed()}: see * https://github.com/apache/incubator-druid/issues/8070. * * @return overshadowed segment Ids list @@ -161,7 +161,7 @@ private List determineOvershadowedSegments() VersionedIntervalTimeline usedSegmentsTimeline = usedSegmentsTimelinesPerDataSource.get(dataSource.getName()); for (DataSegment segment : dataSource.getSegments()) { - if (usedSegmentsTimeline.isOvershadowed(segment.getInterval(), segment.getVersion())) { + if (usedSegmentsTimeline.isOvershadowed(segment.getInterval(), segment.getVersion(), segment)) { overshadowedSegments.add(segment.getId()); } } diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index a9cca5b154a5..f9f4c782cc8d 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -23,10 +23,11 @@ import com.google.common.collect.Sets; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.Overshadowable; import java.util.Set; -public class SegmentLoadInfo +public class SegmentLoadInfo implements Overshadowable { private final DataSegment segment; private final Set servers; @@ -94,4 +95,40 @@ public String toString() ", servers=" + servers + '}'; } + + @Override + public boolean overshadows(SegmentLoadInfo other) + { + return segment.overshadows(other.segment); + } + + @Override + public int getStartRootPartitionId() + { + return segment.getStartRootPartitionId(); + } + + @Override + public int getEndRootPartitionId() + { + return segment.getEndRootPartitionId(); + } + + @Override + public String getVersion() + { + return segment.getVersion(); + } + + @Override + public short getMinorVersion() + { + return segment.getMinorVersion(); + } + + @Override + public short getAtomicUpdateGroupSize() + { + return segment.getAtomicUpdateGroupSize(); + } } diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index ac4fb84b477d..242908d33505 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -24,6 +24,7 @@ import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.Overshadowable; import javax.annotation.Nullable; import java.util.ArrayList; @@ -35,7 +36,7 @@ /** */ -public class ServerSelector implements DiscoverySelector +public class ServerSelector implements DiscoverySelector, Overshadowable { private final Int2ObjectRBTreeMap> historicalServers; @@ -169,4 +170,42 @@ public QueryableDruidServer pick() return strategy.pick(realtimeServers, segment.get()); } } + + @Override + public boolean overshadows(ServerSelector other) + { + final DataSegment thisSegment = segment.get(); + final DataSegment thatSegment = other.getSegment(); + return thisSegment.overshadows(thatSegment); + } + + @Override + public int getStartRootPartitionId() + { + return segment.get().getStartRootPartitionId(); + } + + @Override + public int getEndRootPartitionId() + { + return segment.get().getEndRootPartitionId(); + } + + @Override + public String getVersion() + { + return segment.get().getVersion(); + } + + @Override + public short getMinorVersion() + { + return segment.get().getMinorVersion(); + } + + @Override + public short getAtomicUpdateGroupSize() + { + return segment.get().getAtomicUpdateGroupSize(); + } } diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java index 95c1fce15c6e..9eacec56bebe 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java @@ -22,8 +22,10 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; import java.util.List; @@ -93,6 +95,8 @@ default List getUsedSegmentsForInterval(String dataSource, Interval * @param sequenceName name of the group of ingestion tasks producing a segment series * @param previousSegmentId previous segment in the series; may be null or empty, meaning this is the first segment * @param interval interval for which to allocate a segment + * @param shardSpecFactory shardSpecFactory containing all necessary information to create a shardSpec for the + * new segmentId * @param maxVersion use this version if we have no better version to use. The returned segment identifier may * have a version lower than this one, but will not have one higher. * @param skipSegmentLineageCheck if true, perform lineage validation using previousSegmentId for this sequence. @@ -103,8 +107,9 @@ default List getUsedSegmentsForInterval(String dataSource, Interval SegmentIdWithShardSpec allocatePendingSegment( String dataSource, String sequenceName, - String previousSegmentId, + @Nullable String previousSegmentId, Interval interval, + ShardSpecFactory shardSpecFactory, String maxVersion, boolean skipSegmentLineageCheck ); @@ -144,8 +149,8 @@ SegmentIdWithShardSpec allocatePendingSegment( */ SegmentPublishResult announceHistoricalSegments( Set segments, - DataSourceMetadata startMetadata, - DataSourceMetadata endMetadata + @Nullable DataSourceMetadata startMetadata, + @Nullable DataSourceMetadata endMetadata ) throws IOException; /** diff --git a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java index c77dee153af0..44e2196ce301 100644 --- a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java +++ b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java @@ -44,10 +44,10 @@ import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.TimelineObjectHolder; import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NoneShardSpec; -import org.apache.druid.timeline.partition.NumberedShardSpec; import org.apache.druid.timeline.partition.PartitionChunk; +import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.joda.time.Interval; import org.skife.jdbi.v2.FoldController; import org.skife.jdbi.v2.Folder3; @@ -68,6 +68,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -249,8 +250,8 @@ public Set announceHistoricalSegments(final Set segmen @Override public SegmentPublishResult announceHistoricalSegments( final Set segments, - final DataSourceMetadata startMetadata, - final DataSourceMetadata endMetadata + @Nullable final DataSourceMetadata startMetadata, + @Nullable final DataSourceMetadata endMetadata ) throws IOException { if (segments.isEmpty()) { @@ -345,6 +346,7 @@ public SegmentIdWithShardSpec allocatePendingSegment( final String sequenceName, @Nullable final String previousSegmentId, final Interval interval, + final ShardSpecFactory shardSpecFactory, final String maxVersion, final boolean skipSegmentLineageCheck ) @@ -352,12 +354,19 @@ public SegmentIdWithShardSpec allocatePendingSegment( Preconditions.checkNotNull(dataSource, "dataSource"); Preconditions.checkNotNull(sequenceName, "sequenceName"); Preconditions.checkNotNull(interval, "interval"); - Preconditions.checkNotNull(maxVersion, "maxVersion"); + Preconditions.checkNotNull(maxVersion, "version"); return connector.retryWithHandle( handle -> { if (skipSegmentLineageCheck) { - return allocatePendingSegment(handle, dataSource, sequenceName, interval, maxVersion); + return allocatePendingSegment( + handle, + dataSource, + sequenceName, + interval, + shardSpecFactory, + maxVersion + ); } else { return allocatePendingSegmentWithSegmentLineageCheck( handle, @@ -365,6 +374,7 @@ public SegmentIdWithShardSpec allocatePendingSegment( sequenceName, previousSegmentId, interval, + shardSpecFactory, maxVersion ); } @@ -379,6 +389,7 @@ private SegmentIdWithShardSpec allocatePendingSegmentWithSegmentLineageCheck( final String sequenceName, @Nullable final String previousSegmentId, final Interval interval, + final ShardSpecFactory shardSpecFactory, final String maxVersion ) throws IOException { @@ -406,7 +417,13 @@ private SegmentIdWithShardSpec allocatePendingSegmentWithSegmentLineageCheck( return result.segmentIdentifier; } - final SegmentIdWithShardSpec newIdentifier = createNewSegment(handle, dataSource, interval, maxVersion); + final SegmentIdWithShardSpec newIdentifier = createNewSegment( + handle, + dataSource, + interval, + shardSpecFactory, + maxVersion + ); if (newIdentifier == null) { return null; } @@ -446,6 +463,7 @@ private SegmentIdWithShardSpec allocatePendingSegment( final String dataSource, final String sequenceName, final Interval interval, + final ShardSpecFactory shardSpecFactory, final String maxVersion ) throws IOException { @@ -475,7 +493,13 @@ private SegmentIdWithShardSpec allocatePendingSegment( return result.segmentIdentifier; } - final SegmentIdWithShardSpec newIdentifier = createNewSegment(handle, dataSource, interval, maxVersion); + final SegmentIdWithShardSpec newIdentifier = createNewSegment( + handle, + dataSource, + interval, + shardSpecFactory, + maxVersion + ); if (newIdentifier == null) { return null; } @@ -614,13 +638,10 @@ private SegmentIdWithShardSpec createNewSegment( final Handle handle, final String dataSource, final Interval interval, + final ShardSpecFactory shardSpecFactory, final String maxVersion ) throws IOException { - // Make up a pending segment based on existing segments and pending segments in the DB. This works - // assuming that all tasks inserting segments at a particular point in time are going through the - // allocatePendingSegment flow. This should be assured through some other mechanism (like task locks). - final List> existingChunks = getTimelineForIntervalsWithHandle( handle, dataSource, @@ -630,24 +651,36 @@ private SegmentIdWithShardSpec createNewSegment( if (existingChunks.size() > 1) { // Not possible to expand more than one chunk with a single segment. log.warn( - "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: already have [%,d] chunks.", + "Cannot allocate new segment for dataSource[%s], interval[%s]: already have [%,d] chunks.", dataSource, interval, - maxVersion, existingChunks.size() ); return null; + } else { + if (existingChunks + .stream() + .flatMap(holder -> StreamSupport.stream(holder.getObject().spliterator(), false)) + .anyMatch(chunk -> !chunk.getObject().getShardSpec().isCompatible(shardSpecFactory.getShardSpecClass()))) { + // All existing segments should have a compatible shardSpec with shardSpecFactory. + return null; + } + + // max partitionId of the SAME shardSpec SegmentIdWithShardSpec maxId = null; if (!existingChunks.isEmpty()) { TimelineObjectHolder existingHolder = Iterables.getOnlyElement(existingChunks); - for (PartitionChunk existing : existingHolder.getObject()) { - if (maxId == null || - maxId.getShardSpec().getPartitionNum() < existing.getObject().getShardSpec().getPartitionNum()) { - maxId = SegmentIdWithShardSpec.fromDataSegment(existing.getObject()); - } - } + + maxId = StreamSupport.stream(existingHolder.getObject().spliterator(), false) + // Here we check only the segments of the same shardSpec to find out the max partitionId. + // Note that OverwriteShardSpec has the higher range for partitionId than others. + // See PartitionIds. + .filter(chunk -> chunk.getObject().getShardSpec().getClass() == shardSpecFactory.getShardSpecClass()) + .max(Comparator.comparing(chunk -> chunk.getObject().getShardSpec().getPartitionNum())) + .map(chunk -> SegmentIdWithShardSpec.fromDataSegment(chunk.getObject())) + .orElse(null); } final List pendings = getPendingSegmentsForIntervalWithHandle( @@ -656,22 +689,35 @@ private SegmentIdWithShardSpec createNewSegment( interval ); - for (SegmentIdWithShardSpec pending : pendings) { - if (maxId == null || - pending.getVersion().compareTo(maxId.getVersion()) > 0 || - (pending.getVersion().equals(maxId.getVersion()) - && pending.getShardSpec().getPartitionNum() > maxId.getShardSpec().getPartitionNum())) { - maxId = pending; - } + if (maxId != null) { + pendings.add(maxId); + } + + maxId = pendings.stream() + .filter(id -> id.getShardSpec().getClass() == shardSpecFactory.getShardSpecClass()) + .max((id1, id2) -> { + final int versionCompare = id1.getVersion().compareTo(id2.getVersion()); + if (versionCompare != 0) { + return versionCompare; + } else { + return Integer.compare(id1.getShardSpec().getPartitionNum(), id2.getShardSpec().getPartitionNum()); + } + }) + .orElse(null); + + // Find the major version of existing segments + @Nullable final String versionOfExistingChunks; + if (!existingChunks.isEmpty()) { + versionOfExistingChunks = existingChunks.get(0).getVersion(); + } else if (!pendings.isEmpty()) { + versionOfExistingChunks = pendings.get(0).getVersion(); + } else { + versionOfExistingChunks = null; } if (maxId == null) { - return new SegmentIdWithShardSpec( - dataSource, - interval, - maxVersion, - new NumberedShardSpec(0, 0) - ); + final ShardSpec shardSpec = shardSpecFactory.create(jsonMapper, null); + return new SegmentIdWithShardSpec(dataSource, interval, versionOfExistingChunks == null ? maxVersion : versionOfExistingChunks, shardSpec); } else if (!maxId.getInterval().equals(interval) || maxId.getVersion().compareTo(maxVersion) > 0) { log.warn( "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: conflicting segment[%s].", @@ -681,33 +727,14 @@ private SegmentIdWithShardSpec createNewSegment( maxId ); return null; - } else if (maxId.getShardSpec() instanceof LinearShardSpec) { - return new SegmentIdWithShardSpec( - dataSource, - maxId.getInterval(), - maxId.getVersion(), - new LinearShardSpec(maxId.getShardSpec().getPartitionNum() + 1) - ); - } else if (maxId.getShardSpec() instanceof NumberedShardSpec) { + } else { + final ShardSpec newShardSpec = shardSpecFactory.create(jsonMapper, maxId.getShardSpec()); return new SegmentIdWithShardSpec( dataSource, maxId.getInterval(), - maxId.getVersion(), - new NumberedShardSpec( - maxId.getShardSpec().getPartitionNum() + 1, - ((NumberedShardSpec) maxId.getShardSpec()).getPartitions() - ) + Preconditions.checkNotNull(versionOfExistingChunks, "versionOfExistingChunks"), + newShardSpec ); - } else { - log.warn( - "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: ShardSpec class[%s] used by [%s].", - dataSource, - interval, - maxVersion, - maxId.getShardSpec().getClass(), - maxId - ); - return null; } } } @@ -751,7 +778,7 @@ private boolean announceHistoricalSegment( // SELECT -> INSERT can fail due to races; callers must be prepared to retry. // Avoiding ON DUPLICATE KEY since it's not portable. // Avoiding try/catch since it may cause inadvertent transaction-splitting. - handle.createStatement( + final int numRowsInserted = handle.createStatement( StringUtils.format( "INSERT INTO %1$s (id, dataSource, created_date, start, %2$send%2$s, partitioned, version, used, payload) " + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload)", @@ -770,7 +797,13 @@ private boolean announceHistoricalSegment( .bind("payload", jsonMapper.writeValueAsBytes(segment)) .execute(); - log.info("Published segment [%s] to DB with used flag [%s]", segment.getId(), used); + if (numRowsInserted == 1) { + log.info("Published segment [%s] to DB with used flag [%s], json[%s]", segment.getId(), used, jsonMapper.writeValueAsString(segment)); + } else if (numRowsInserted == 0) { + throw new ISE("Failed to publish segment[%s] to DB with used flag[%s], json[%s]", segment.getId(), used, jsonMapper.writeValueAsString(segment)); + } else { + throw new ISE("WTH? numRowsInserted[%s] is larger than 1 after inserting segment[%s] with used flag[%s], json[%s]", numRowsInserted, segment.getId(), used, jsonMapper.writeValueAsString(segment)); + } } catch (Exception e) { log.error(e, "Exception inserting segment [%s] with used flag [%s] into DB", segment.getId(), used); diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataSegmentManager.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataSegmentManager.java index 2f37fd6ed719..4f22478cd36a 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataSegmentManager.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataSegmentManager.java @@ -546,7 +546,7 @@ private int markNonOvershadowedSegmentsAsUsed( { List segmentIdsToMarkAsUsed = new ArrayList<>(); for (DataSegment segment : unusedSegments) { - if (timeline.isOvershadowed(segment.getInterval(), segment.getVersion())) { + if (timeline.isOvershadowed(segment.getInterval(), segment.getVersion(), segment)) { continue; } segmentIdsToMarkAsUsed.add(segment.getId().toString()); diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/AppenderatorDriverSegmentLockHelper.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/AppenderatorDriverSegmentLockHelper.java new file mode 100644 index 000000000000..2de7089e314d --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/AppenderatorDriverSegmentLockHelper.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.realtime.appenderator; + +/** + * Lock helper for {@link StreamAppenderatorDriver}. It's only used to lock segments on restart of the driver. + */ +public interface AppenderatorDriverSegmentLockHelper +{ + AppenderatorDriverSegmentLockHelper NOOP = segmentId -> true; + + boolean lock(SegmentIdWithShardSpec segmentId); +} diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/AppenderatorImpl.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/AppenderatorImpl.java index 136b24e84433..1a2fd5f63148 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/AppenderatorImpl.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/AppenderatorImpl.java @@ -271,7 +271,7 @@ public AppenderatorAddResult add( boolean isPersistRequired = false; boolean persist = false; - List persistReasons = new ArrayList(); + List persistReasons = new ArrayList<>(); if (!sink.canAppendRow()) { persist = true; @@ -626,7 +626,11 @@ public ListenableFuture push( continue; } - final DataSegment dataSegment = mergeAndPush(entry.getKey(), entry.getValue(), useUniquePath); + final DataSegment dataSegment = mergeAndPush( + entry.getKey(), + entry.getValue(), + useUniquePath + ); if (dataSegment != null) { dataSegments.add(dataSegment); } else { @@ -745,7 +749,8 @@ private DataSegment mergeAndPush(final SegmentIdWithShardSpec identifier, final // semantics. () -> dataSegmentPusher.push( mergedFile, - sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)), + sink.getSegment() + .withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)), useUniquePath ), exception -> exception instanceof Exception, diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/Appenderators.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/Appenderators.java index 44d91cb54416..d58772ade6a2 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/Appenderators.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/Appenderators.java @@ -23,7 +23,6 @@ import org.apache.druid.client.cache.Cache; import org.apache.druid.client.cache.CacheConfig; import org.apache.druid.client.cache.CachePopulatorStats; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.IndexIO; @@ -33,8 +32,6 @@ import org.apache.druid.segment.realtime.FireDepartmentMetrics; import org.apache.druid.server.coordination.DataSegmentAnnouncer; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.partition.ShardSpec; -import org.joda.time.Interval; import java.util.concurrent.ExecutorService; @@ -127,9 +124,4 @@ public void unannounceSegments(Iterable segments) null ); } - - public static String getSequenceName(Interval interval, String version, ShardSpec shardSpec) - { - return StringUtils.format("index_%s_%s_%d", interval, version, shardSpec.getPartitionNum()); - } } diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java index 9ef242097bbc..4a9734246148 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java @@ -272,7 +272,7 @@ Map getSegments() * @return currently persisted commit metadata */ @Nullable - public abstract Object startJob(); + public abstract Object startJob(AppenderatorDriverSegmentLockHelper lockHelper); /** * Find a segment in the {@link SegmentState#APPENDING} state for the given timestamp and sequenceName. @@ -539,6 +539,7 @@ ListenableFuture dropInBackground(SegmentsAndMetadata segme * @return a future for publishing segments */ ListenableFuture publishInBackground( + @Nullable Set segmentsToBeOverwritten, SegmentsAndMetadata segmentsAndMetadata, TransactionalSegmentPublisher publisher ) @@ -558,6 +559,7 @@ ListenableFuture publishInBackground( final Object metadata = segmentsAndMetadata.getCommitMetadata(); final ImmutableSet ourSegments = ImmutableSet.copyOf(segmentsAndMetadata.getSegments()); final SegmentPublishResult publishResult = publisher.publishSegments( + segmentsToBeOverwritten, ourSegments, metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() ); diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java index 3b084f067b5a..e9e946bbb74b 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -72,6 +73,12 @@ public BatchAppenderatorDriver( super(appenderator, segmentAllocator, usedSegmentChecker, dataSegmentKiller); } + @Nullable + public Object startJob() + { + return startJob(AppenderatorDriverSegmentLockHelper.NOOP); + } + /** * This method always returns null because batch ingestion doesn't support restoring tasks on failures. * @@ -79,7 +86,7 @@ public BatchAppenderatorDriver( */ @Override @Nullable - public Object startJob() + public Object startJob(AppenderatorDriverSegmentLockHelper lockHelper) { final Object metadata = appenderator.startJob(); if (metadata != null) { @@ -129,11 +136,12 @@ private SegmentsAndMetadata pushAndClear( long pushAndClearTimeoutMs ) throws InterruptedException, ExecutionException, TimeoutException { - final Map requestedSegmentIdsForSequences = getAppendingSegments(sequenceNames) - .collect(Collectors.toMap(SegmentWithState::getSegmentIdentifier, Function.identity())); + final Set requestedSegmentIdsForSequences = getAppendingSegments(sequenceNames) + .map(SegmentWithState::getSegmentIdentifier) + .collect(Collectors.toSet()); final ListenableFuture future = ListenableFutures.transformAsync( - pushInBackground(null, requestedSegmentIdsForSequences.keySet(), false), + pushInBackground(null, requestedSegmentIdsForSequences, false), this::dropInBackground ); @@ -147,11 +155,11 @@ private SegmentsAndMetadata pushAndClear( .stream() .collect(Collectors.toMap(SegmentIdWithShardSpec::fromDataSegment, Function.identity())); - if (!pushedSegmentIdToSegmentMap.keySet().equals(requestedSegmentIdsForSequences.keySet())) { + if (!pushedSegmentIdToSegmentMap.keySet().equals(requestedSegmentIdsForSequences)) { throw new ISE( "Pushed segments[%s] are different from the requested ones[%s]", pushedSegmentIdToSegmentMap.keySet(), - requestedSegmentIdsForSequences.keySet() + requestedSegmentIdsForSequences ); } @@ -184,11 +192,15 @@ private SegmentsAndMetadata pushAndClear( /** * Publish all segments. * - * @param publisher segment publisher + * @param segmentsToBeOverwritten segments which can be overwritten by new segments published by the given publisher + * @param publisher segment publisher * * @return a {@link ListenableFuture} for the publish task */ - public ListenableFuture publishAll(final TransactionalSegmentPublisher publisher) + public ListenableFuture publishAll( + @Nullable final Set segmentsToBeOverwritten, + final TransactionalSegmentPublisher publisher + ) { final Map snapshot; synchronized (segments) { @@ -196,6 +208,7 @@ public ListenableFuture publishAll(final TransactionalSegme } return publishInBackground( + segmentsToBeOverwritten, new SegmentsAndMetadata( snapshot .values() diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/SegmentIdWithShardSpec.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/SegmentIdWithShardSpec.java index 58dad0d36448..6e85794e69d1 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/SegmentIdWithShardSpec.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/SegmentIdWithShardSpec.java @@ -59,6 +59,16 @@ public SegmentId asSegmentId() return id; } + public SegmentIdWithShardSpec withShardSpec(ShardSpec shardSpec) + { + return new SegmentIdWithShardSpec( + id.getDataSource(), + id.getInterval(), + id.getVersion(), + shardSpec + ); + } + @JsonProperty public String getDataSource() { @@ -83,6 +93,11 @@ public ShardSpec getShardSpec() return shardSpec; } + public String getIdentifierAsString() + { + return asString; + } + @Override public boolean equals(Object o) { diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java index 2599387dd158..c68b4a9bb270 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java @@ -107,7 +107,7 @@ public StreamAppenderatorDriver( @Override @Nullable - public Object startJob() + public Object startJob(AppenderatorDriverSegmentLockHelper lockHelper) { handoffNotifier.start(); @@ -133,6 +133,14 @@ public Object startJob() final SegmentsForSequenceBuilder builder = new SegmentsForSequenceBuilder(lastSegmentIds.get(sequenceName)); builders.put(sequenceName, builder); entry.getValue().forEach(builder::add); + if (lockHelper != null) { + for (SegmentWithState segmentWithState : entry.getValue()) { + if (segmentWithState.getState() != SegmentState.PUSHED_AND_DROPPED + && !lockHelper.lock(segmentWithState.getSegmentIdentifier())) { + throw new ISE("Failed to lock segment[%s]", segmentWithState.getSegmentIdentifier()); + } + } + } } builders.forEach((sequence, builder) -> segments.put(sequence, builder.build())); @@ -273,6 +281,7 @@ public ListenableFuture publish( // version of a segment with the same identifier containing different data; see DataSegmentPusher.push() docs pushInBackground(wrapCommitter(committer), theSegments, true), sam -> publishInBackground( + null, sam, publisher ) diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/TransactionalSegmentPublisher.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/TransactionalSegmentPublisher.java index 0749a2892036..8ebc66237935 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/TransactionalSegmentPublisher.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/TransactionalSegmentPublisher.java @@ -20,11 +20,21 @@ package org.apache.druid.segment.realtime.appenderator; import org.apache.druid.indexing.overlord.SegmentPublishResult; +import org.apache.druid.java.util.common.ISE; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.OverwriteShardSpec; +import org.joda.time.Interval; import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; public interface TransactionalSegmentPublisher { @@ -38,8 +48,71 @@ public interface TransactionalSegmentPublisher * @throws IOException if there was an I/O error when publishing * @throws RuntimeException if we cannot tell if the segments were published or not, for some other reason */ - SegmentPublishResult publishSegments( - Set segments, + SegmentPublishResult publishAnnotatedSegments( + @Nullable Set segmentsToBeOverwritten, + Set segmentsToPublish, @Nullable Object commitMetadata ) throws IOException; + + default SegmentPublishResult publishSegments( + @Nullable Set segmentsToBeOverwritten, + Set segmentsToPublish, + @Nullable Object commitMetadata + ) + throws IOException + { + return publishAnnotatedSegments( + segmentsToBeOverwritten, + annotateAtomicUpdateGroupSize(segmentsToPublish), + commitMetadata + ); + } + + static Set annotateAtomicUpdateGroupSize(Set segments) + { + final Map> intervalToSegments = new HashMap<>(); + segments.forEach( + segment -> intervalToSegments.computeIfAbsent(segment.getInterval(), k -> new ArrayList<>()).add(segment) + ); + + for (Entry> entry : intervalToSegments.entrySet()) { + final Interval interval = entry.getKey(); + final List segmentsPerInterval = entry.getValue(); + final boolean isNonFirstGeneration = segmentsPerInterval.get(0).getShardSpec() instanceof OverwriteShardSpec; + + final boolean anyMismatch = segmentsPerInterval.stream().anyMatch( + segment -> (segment.getShardSpec() instanceof OverwriteShardSpec) != isNonFirstGeneration + ); + if (anyMismatch) { + throw new ISE( + "WTH? some segments have empty overshadwedSegments but others are not? " + + "segments with non-overwritingShardSpec: [%s]," + + "segments with overwritingShardSpec: [%s]", + segmentsPerInterval.stream() + .filter(segment -> !(segment.getShardSpec() instanceof OverwriteShardSpec)) + .collect(Collectors.toList()), + segmentsPerInterval.stream() + .filter(segment -> segment.getShardSpec() instanceof OverwriteShardSpec) + .collect(Collectors.toList()) + ); + } + + if (isNonFirstGeneration) { + // The segments which are published together consist an atomicUpdateGroup. + + intervalToSegments.put( + interval, + segmentsPerInterval + .stream() + .map(segment -> { + final OverwriteShardSpec shardSpec = (OverwriteShardSpec) segment.getShardSpec(); + return segment.withShardSpec(shardSpec.withAtomicUpdateGroupSize((short) segmentsPerInterval.size())); + }) + .collect(Collectors.toList()) + ); + } + } + + return intervalToSegments.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); + } } diff --git a/server/src/main/java/org/apache/druid/segment/realtime/plumber/RealtimePlumber.java b/server/src/main/java/org/apache/druid/segment/realtime/plumber/RealtimePlumber.java index 7aa59f6d8ccd..98dcdfa88951 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/plumber/RealtimePlumber.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/plumber/RealtimePlumber.java @@ -736,7 +736,7 @@ private void addSink(final Sink sink) sinkTimeline.add( sink.getInterval(), sink.getVersion(), - new SingleElementPartitionChunk(sink) + new SingleElementPartitionChunk<>(sink) ); try { segmentAnnouncer.announceSegment(sink.getSegment()); diff --git a/server/src/main/java/org/apache/druid/segment/realtime/plumber/Sink.java b/server/src/main/java/org/apache/druid/segment/realtime/plumber/Sink.java index 4e0c596cbdca..d3218fbc0248 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/plumber/Sink.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/plumber/Sink.java @@ -37,6 +37,7 @@ import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.realtime.FireHydrant; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.Overshadowable; import org.apache.druid.timeline.partition.ShardSpec; import org.joda.time.Interval; @@ -52,7 +53,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; -public class Sink implements Iterable +public class Sink implements Iterable, Overshadowable { private static final IncrementalIndexAddResult ALREADY_SWAPPED = new IncrementalIndexAddResult(-1, -1, null, "write after index swapped"); @@ -142,11 +143,6 @@ public void clearDedupCache() dedupSet.clear(); } - public String getVersion() - { - return version; - } - public Interval getInterval() { return interval; @@ -408,4 +404,42 @@ public String toString() ", schema=" + schema + '}'; } + + @Override + public boolean overshadows(Sink other) + { + // Sink is currently used in timeline only for querying stream data. + // In this case, sinks never overshadow each other. + return false; + } + + @Override + public int getStartRootPartitionId() + { + return shardSpec.getStartRootPartitionId(); + } + + @Override + public int getEndRootPartitionId() + { + return shardSpec.getEndRootPartitionId(); + } + + @Override + public String getVersion() + { + return version; + } + + @Override + public short getMinorVersion() + { + return shardSpec.getMinorVersion(); + } + + @Override + public short getAtomicUpdateGroupSize() + { + return shardSpec.getAtomicUpdateGroupSize(); + } } diff --git a/server/src/main/java/org/apache/druid/server/SegmentManager.java b/server/src/main/java/org/apache/druid/server/SegmentManager.java index 529a37f68342..73a9de1e6c02 100644 --- a/server/src/main/java/org/apache/druid/server/SegmentManager.java +++ b/server/src/main/java/org/apache/druid/server/SegmentManager.java @@ -32,6 +32,7 @@ import org.apache.druid.timeline.VersionedIntervalTimeline; import org.apache.druid.timeline.partition.PartitionChunk; import org.apache.druid.timeline.partition.PartitionHolder; +import org.apache.druid.timeline.partition.ShardSpec; import org.apache.druid.utils.CollectionUtils; import javax.annotation.Nullable; @@ -171,10 +172,11 @@ public boolean loadSegment(final DataSegment segment) throws SegmentLoadingExcep log.warn("Told to load an adapter for segment[%s] that already exists", segment.getId()); resultSupplier.set(false); } else { + final ReferenceCountingSegment referenceCountingSegment = new ReferenceCountingSegment(adapter); loadedIntervals.add( segment.getInterval(), segment.getVersion(), - segment.getShardSpec().createChunk(new ReferenceCountingSegment(adapter)) + segment.getShardSpec().createChunk(referenceCountingSegment) ); dataSourceState.addSegment(segment); resultSupplier.set(true); @@ -213,16 +215,26 @@ public void dropSegment(final DataSegment segment) (dataSourceName, dataSourceState) -> { if (dataSourceState == null) { log.info("Told to delete a queryable for a dataSource[%s] that doesn't exist.", dataSourceName); + return null; } else { final VersionedIntervalTimeline loadedIntervals = dataSourceState.getTimeline(); + final ShardSpec shardSpec = segment.getShardSpec(); final PartitionChunk removed = loadedIntervals.remove( segment.getInterval(), segment.getVersion(), // remove() internally searches for a partitionChunk to remove which is *equal* to the given // partitionChunk. Note that partitionChunk.equals() checks only the partitionNum, but not the object. - segment.getShardSpec().createChunk(null) + segment.getShardSpec().createChunk( + new ReferenceCountingSegment( + null, + shardSpec.getStartRootPartitionId(), + shardSpec.getEndRootPartitionId(), + shardSpec.getMinorVersion(), + shardSpec.getAtomicUpdateGroupSize() + ) + ) ); final ReferenceCountingSegment oldQueryable = (removed == null) ? null : removed.getObject(); @@ -239,10 +251,10 @@ public void dropSegment(final DataSegment segment) segment.getVersion() ); } - } - // Returning null removes the entry of dataSource from the map - return dataSourceState == null || dataSourceState.isEmpty() ? null : dataSourceState; + // Returning null removes the entry of dataSource from the map + return dataSourceState.isEmpty() ? null : dataSourceState; + } } ); diff --git a/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorCleanupOvershadowed.java b/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorCleanupOvershadowed.java index bde7c15810d2..9845991c7b79 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorCleanupOvershadowed.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorCleanupOvershadowed.java @@ -74,7 +74,8 @@ public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) // Mark all segments as unused in db that are overshadowed by served segments for (DataSegment dataSegment : params.getUsedSegments()) { VersionedIntervalTimeline timeline = timelines.get(dataSegment.getDataSource()); - if (timeline != null && timeline.isOvershadowed(dataSegment.getInterval(), dataSegment.getVersion())) { + if (timeline != null + && timeline.isOvershadowed(dataSegment.getInterval(), dataSegment.getVersion(), dataSegment)) { coordinator.markSegmentAsUnused(dataSegment); stats.addToGlobalStat("overShadowedCount", 1); } diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java index 8974cc1990bf..127627d086af 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java @@ -1761,11 +1761,15 @@ private ServerSelector makeMockSingleDimensionSelector( int partitionNum ) { - DataSegment segment = EasyMock.createNiceMock(DataSegment.class); - EasyMock.expect(segment.getId()).andReturn(SegmentId.dummy(DATA_SOURCE)).anyTimes(); - EasyMock.expect(segment.getShardSpec()).andReturn(new SingleDimensionShardSpec(dimension, start, end, partitionNum)) - .anyTimes(); - EasyMock.replay(segment); + final DataSegment segment = new DataSegment( + SegmentId.dummy(DATA_SOURCE), + null, + null, + null, + new SingleDimensionShardSpec(dimension, start, end, partitionNum), + 9, + 0L + ); ServerSelector selector = new ServerSelector( segment, @@ -2201,11 +2205,26 @@ private List> populateTimeline( serverExpectations .computeIfAbsent(lastServer, server -> new ServerExpectations(server, makeMock(mocks, QueryRunner.class))); + final ShardSpec shardSpec; + if (numChunks == 1) { + shardSpec = new SingleDimensionShardSpec("dimAll", null, null, 0); + } else { + String start = null; + String end = null; + if (j > 0) { + start = String.valueOf(j); + } + if (j + 1 < numChunks) { + end = String.valueOf(j + 1); + } + shardSpec = new SingleDimensionShardSpec("dim" + k, start, end, j); + } DataSegment mockSegment = makeMock(mocks, DataSegment.class); ServerExpectation expectation = new ServerExpectation<>( SegmentId.dummy(StringUtils.format("%s_%s", k, j)), // interval/chunk queryIntervals.get(k), mockSegment, + shardSpec, expectedResults.get(k).get(j) ); serverExpectations.get(lastServer).addExpectation(expectation); @@ -2216,20 +2235,6 @@ private List> populateTimeline( new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()) ); selector.addServerAndUpdateSegment(new QueryableDruidServer(lastServer, null), selector.getSegment()); - final ShardSpec shardSpec; - if (numChunks == 1) { - shardSpec = new SingleDimensionShardSpec("dimAll", null, null, 0); - } else { - String start = null; - String end = null; - if (j > 0) { - start = String.valueOf(j); - } - if (j + 1 < numChunks) { - end = String.valueOf(j + 1); - } - shardSpec = new SingleDimensionShardSpec("dim" + k, start, end, j); - } EasyMock.reset(mockSegment); EasyMock.expect(mockSegment.getShardSpec()) .andReturn(shardSpec) @@ -2743,18 +2748,21 @@ private static class ServerExpectation private final SegmentId segmentId; private final Interval interval; private final DataSegment segment; + private final ShardSpec shardSpec; private final Iterable> results; public ServerExpectation( SegmentId segmentId, Interval interval, DataSegment segment, + ShardSpec shardSpec, Iterable> results ) { this.segmentId = segmentId; this.interval = interval; this.segment = segment; + this.shardSpec = shardSpec; this.results = results; } @@ -2822,7 +2830,7 @@ public Map getLoadSpec() @JsonProperty public String getVersion() { - return baseSegment.getVersion(); + return "version"; } @Override @@ -2898,6 +2906,43 @@ public String toString() { return baseSegment.toString(); } + + @Override + public int getStartRootPartitionId() + { + return shardSpec.getStartRootPartitionId(); + } + + @Override + public int getEndRootPartitionId() + { + return shardSpec.getEndRootPartitionId(); + } + + @Override + public short getMinorVersion() + { + return shardSpec.getMinorVersion(); + } + + @Override + public short getAtomicUpdateGroupSize() + { + return shardSpec.getAtomicUpdateGroupSize(); + } + + @Override + public boolean overshadows(DataSegment other) + { + if (getDataSource().equals(other.getDataSource()) + && getInterval().overlaps(other.getInterval()) + && getVersion().equals(other.getVersion())) { + return getStartRootPartitionId() <= other.getStartRootPartitionId() + && getEndRootPartitionId() >= other.getEndRootPartitionId() + && getMinorVersion() > other.getMinorVersion(); + } + return false; + } } } diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java index 879ddfb13b4c..76d637c6df6d 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java @@ -32,9 +32,18 @@ import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec; +import org.apache.druid.timeline.partition.HashBasedNumberedShardSpecFactory; import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NoneShardSpec; +import org.apache.druid.timeline.partition.NumberedOverwriteShardSpec; +import org.apache.druid.timeline.partition.NumberedOverwritingShardSpecFactory; import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpecFactory; +import org.apache.druid.timeline.partition.PartitionChunk; +import org.apache.druid.timeline.partition.PartitionIds; +import org.apache.druid.timeline.partition.ShardSpecFactory; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; @@ -48,10 +57,12 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; public class IndexerSQLMetadataStorageCoordinatorTest { @@ -181,7 +192,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest public void setUp() { derbyConnector = derbyConnectorRule.getConnector(); - mapper.registerSubtypes(LinearShardSpec.class, NumberedShardSpec.class); + mapper.registerSubtypes(LinearShardSpec.class, NumberedShardSpec.class, HashBasedNumberedShardSpec.class); derbyConnector.createDataSourceTable(); derbyConnector.createTaskTables(); derbyConnector.createSegmentTable(); @@ -840,6 +851,7 @@ private void additionalNumberedShardTest(Set segments) throws IOExc @Test public void testAllocatePendingSegment() { + final ShardSpecFactory shardSpecFactory = NumberedShardSpecFactory.instance(); final String dataSource = "ds"; final Interval interval = Intervals.of("2017-01-01/2017-02-01"); final SegmentIdWithShardSpec identifier = coordinator.allocatePendingSegment( @@ -847,6 +859,7 @@ public void testAllocatePendingSegment() "seq", null, interval, + shardSpecFactory, "version", false ); @@ -858,6 +871,7 @@ public void testAllocatePendingSegment() "seq", identifier.toString(), interval, + shardSpecFactory, identifier.getVersion(), false ); @@ -869,6 +883,7 @@ public void testAllocatePendingSegment() "seq", identifier1.toString(), interval, + shardSpecFactory, identifier1.getVersion(), false ); @@ -880,6 +895,7 @@ public void testAllocatePendingSegment() "seq", identifier1.toString(), interval, + shardSpecFactory, identifier1.getVersion(), false ); @@ -892,6 +908,7 @@ public void testAllocatePendingSegment() "seq1", null, interval, + shardSpecFactory, "version", false ); @@ -902,6 +919,7 @@ public void testAllocatePendingSegment() @Test public void testDeletePendingSegment() throws InterruptedException { + final ShardSpecFactory shardSpecFactory = NumberedShardSpecFactory.instance(); final String dataSource = "ds"; final Interval interval = Intervals.of("2017-01-01/2017-02-01"); String prevSegmentId = null; @@ -914,6 +932,7 @@ public void testDeletePendingSegment() throws InterruptedException "seq", prevSegmentId, interval, + shardSpecFactory, "version", false ); @@ -928,6 +947,7 @@ public void testDeletePendingSegment() throws InterruptedException "seq", prevSegmentId, interval, + shardSpecFactory, "version", false ); @@ -937,4 +957,165 @@ public void testDeletePendingSegment() throws InterruptedException final int numDeleted = coordinator.deletePendingSegments(dataSource, new Interval(begin, secondBegin)); Assert.assertEquals(10, numDeleted); } + + @Test + public void testAllocatePendingSegmentsWithOvershadowingSegments() throws IOException + { + final String dataSource = "ds"; + final Interval interval = Intervals.of("2017-01-01/2017-02-01"); + String prevSegmentId = null; + + for (int i = 0; i < 10; i++) { + final SegmentIdWithShardSpec identifier = coordinator.allocatePendingSegment( + dataSource, + "seq", + prevSegmentId, + interval, + new NumberedOverwritingShardSpecFactory(0, 1, (short) (i + 1)), + "version", + false + ); + Assert.assertEquals( + StringUtils.format( + "ds_2017-01-01T00:00:00.000Z_2017-02-01T00:00:00.000Z_version%s", + "_" + (i + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID) + ), + identifier.toString() + ); + prevSegmentId = identifier.toString(); + final Set toBeAnnounced = Collections.singleton( + new DataSegment( + identifier.getDataSource(), + identifier.getInterval(), + identifier.getVersion(), + null, + Collections.emptyList(), + Collections.emptyList(), + ((NumberedOverwriteShardSpec) identifier.getShardSpec()).withAtomicUpdateGroupSize(1), + 0, + 10L + ) + ); + final Set announced = coordinator.announceHistoricalSegments(toBeAnnounced); + + Assert.assertEquals(toBeAnnounced, announced); + } + + final VersionedIntervalTimeline timeline = VersionedIntervalTimeline + .forSegments(coordinator.getUsedSegmentsForInterval(dataSource, interval)); + + final List visibleSegments = timeline + .lookup(interval) + .stream() + .flatMap(holder -> StreamSupport.stream(holder.getObject().spliterator(), false)) + .map(PartitionChunk::getObject) + .collect(Collectors.toList()); + + Assert.assertEquals(1, visibleSegments.size()); + Assert.assertEquals( + new DataSegment( + dataSource, + interval, + "version", + null, + Collections.emptyList(), + Collections.emptyList(), + new NumberedOverwriteShardSpec( + 9 + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID, + 0, + 1, + (short) 9, + (short) 1 + ), + 0, + 10L + ), + visibleSegments.get(0) + ); + } + + @Test + public void testAllocatePendingSegmentsForHashBasedNumberedShardSpec() throws IOException + { + final ShardSpecFactory shardSpecFactory = new HashBasedNumberedShardSpecFactory( + null, + 5 + ); + final String dataSource = "ds"; + final Interval interval = Intervals.of("2017-01-01/2017-02-01"); + + SegmentIdWithShardSpec id = coordinator.allocatePendingSegment( + dataSource, + "seq", + null, + interval, + shardSpecFactory, + "version", + true + ); + + HashBasedNumberedShardSpec shardSpec = (HashBasedNumberedShardSpec) id.getShardSpec(); + Assert.assertEquals(0, shardSpec.getPartitionNum()); + Assert.assertEquals(5, shardSpec.getPartitions()); + + coordinator.announceHistoricalSegments( + Collections.singleton( + new DataSegment( + id.getDataSource(), + id.getInterval(), + id.getVersion(), + null, + Collections.emptyList(), + Collections.emptyList(), + id.getShardSpec(), + 0, + 10L + ) + ) + ); + + id = coordinator.allocatePendingSegment( + dataSource, + "seq2", + null, + interval, + shardSpecFactory, + "version", + true + ); + + shardSpec = (HashBasedNumberedShardSpec) id.getShardSpec(); + Assert.assertEquals(1, shardSpec.getPartitionNum()); + Assert.assertEquals(5, shardSpec.getPartitions()); + + coordinator.announceHistoricalSegments( + Collections.singleton( + new DataSegment( + id.getDataSource(), + id.getInterval(), + id.getVersion(), + null, + Collections.emptyList(), + Collections.emptyList(), + id.getShardSpec(), + 0, + 10L + ) + ) + ); + + id = coordinator.allocatePendingSegment( + dataSource, + "seq3", + null, + interval, + new HashBasedNumberedShardSpecFactory(null, 3), + "version", + true + ); + + shardSpec = (HashBasedNumberedShardSpec) id.getShardSpec(); + Assert.assertEquals(2, shardSpec.getPartitionNum()); + Assert.assertEquals(3, shardSpec.getPartitions()); + } } diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java index 6536cb6d4a30..f01495d469e7 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java @@ -103,7 +103,7 @@ public void tearDown() throws Exception @Test public void testSimple() throws Exception { - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); @@ -115,7 +115,7 @@ public void testSimple() throws Exception checkSegmentStates(2, SegmentState.PUSHED_AND_DROPPED); - final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( + final SegmentsAndMetadata published = driver.publishAll(null, makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); @@ -137,7 +137,7 @@ public void testSimple() throws Exception @Test public void testIncrementalPush() throws Exception { - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); int i = 0; for (InputRow row : ROWS) { @@ -151,7 +151,7 @@ public void testIncrementalPush() throws Exception checkSegmentStates(++i, SegmentState.PUSHED_AND_DROPPED); } - final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( + final SegmentsAndMetadata published = driver.publishAll(null, makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); @@ -174,11 +174,11 @@ public void testIncrementalPush() throws Exception @Test public void testRestart() { - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); driver.close(); appenderatorTester.getAppenderator().close(); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); } private void checkSegmentStates(int expectedNumSegmentsInState, SegmentState expectedState) @@ -195,6 +195,6 @@ private void checkSegmentStates(int expectedNumSegmentsInState, SegmentState exp static TransactionalSegmentPublisher makeOkPublisher() { - return (segments, commitMetadata) -> SegmentPublishResult.ok(ImmutableSet.of()); + return (segmentsToBeOverwritten, segmentsToPublish, commitMetadata) -> SegmentPublishResult.ok(ImmutableSet.of()); } } diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java index 9d922537159b..bbbc64a99041 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java @@ -137,12 +137,12 @@ public void testFailDuringPersist() throws IOException, InterruptedException, Ti new FireDepartmentMetrics() ); - driver.startJob(); + driver.startJob(null); final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); segmentHandoffNotifierFactory.setHandoffDelay(100); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); @@ -175,12 +175,12 @@ public void testFailDuringPush() throws IOException, InterruptedException, Timeo new FireDepartmentMetrics() ); - driver.startJob(); + driver.startJob(null); final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); segmentHandoffNotifierFactory.setHandoffDelay(100); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); @@ -213,12 +213,12 @@ public void testFailDuringDrop() throws IOException, InterruptedException, Timeo new FireDepartmentMetrics() ); - driver.startJob(); + driver.startJob(null); final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); segmentHandoffNotifierFactory.setHandoffDelay(100); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); @@ -266,12 +266,12 @@ private void testFailDuringPublishInternal(boolean failWithException) throws Exc new FireDepartmentMetrics() ); - driver.startJob(); + driver.startJob(null); final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); segmentHandoffNotifierFactory.setHandoffDelay(100); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java index c7475d9f722e..0dbe905ad8f5 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java @@ -133,7 +133,7 @@ public void testSimple() throws Exception { final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); @@ -169,7 +169,7 @@ public void testMaxRowsPerSegment() throws Exception { final int numSegments = 3; final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); for (int i = 0; i < numSegments * MAX_ROWS_PER_SEGMENT; i++) { committerSupplier.setMetadata(i + 1); @@ -212,7 +212,7 @@ public void testHandoffTimeout() throws Exception final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); segmentHandoffNotifierFactory.disableHandoff(); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); @@ -237,7 +237,7 @@ public void testPublishPerRow() throws IOException, InterruptedException, Timeou { final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); // Add the first row and publish immediately { @@ -305,7 +305,7 @@ public void testIncrementalHandoff() throws Exception { final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); - Assert.assertNull(driver.startJob()); + Assert.assertNull(driver.startJob(null)); committerSupplier.setMetadata(1); Assert.assertTrue(driver.add(ROWS.get(0), "sequence_0", committerSupplier, false, true).isOk()); @@ -361,12 +361,13 @@ private Set asIdentifiers(Iterable segments static TransactionalSegmentPublisher makeOkPublisher() { - return (segments, commitMetadata) -> SegmentPublishResult.ok(Collections.emptySet()); + return (segmentsToBeOverwritten, segmentsToPublish, commitMetadata) -> + SegmentPublishResult.ok(Collections.emptySet()); } static TransactionalSegmentPublisher makeFailingPublisher(boolean failWithException) { - return (segments, commitMetadata) -> { + return (segmentsToBeOverwritten, segmentsToPublish, commitMetadata) -> { final RuntimeException exception = new RuntimeException("test"); if (failWithException) { throw exception; diff --git a/server/src/test/java/org/apache/druid/server/coordinator/helper/NewestSegmentFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/coordinator/helper/NewestSegmentFirstPolicyTest.java index b28bd7faf98f..4ad8b0da8609 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/helper/NewestSegmentFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/helper/NewestSegmentFirstPolicyTest.java @@ -644,7 +644,7 @@ private static VersionedIntervalTimeline createTimeline( } for (int i = 0; i < spec.numSegmentsPerShard; i++) { - final ShardSpec shardSpec = new NumberedShardSpec(spec.numSegmentsPerShard, i); + final ShardSpec shardSpec = new NumberedShardSpec(i, spec.numSegmentsPerShard); final DataSegment segment = new DataSegment( DATA_SOURCE, segmentInterval, diff --git a/server/src/test/java/org/apache/druid/server/shard/NumberedShardSpecTest.java b/server/src/test/java/org/apache/druid/server/shard/NumberedShardSpecTest.java index 497a6d73a77b..bfd72b4b8851 100644 --- a/server/src/test/java/org/apache/druid/server/shard/NumberedShardSpecTest.java +++ b/server/src/test/java/org/apache/druid/server/shard/NumberedShardSpecTest.java @@ -26,8 +26,10 @@ import com.google.common.collect.Ordering; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.server.ServerTestHelper; +import org.apache.druid.timeline.Overshadowable; import org.apache.druid.timeline.TimelineObjectHolder; import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.NumberedOverwriteShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.apache.druid.timeline.partition.PartitionChunk; import org.apache.druid.timeline.partition.ShardSpec; @@ -38,6 +40,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; public class NumberedShardSpecTest @@ -113,11 +116,14 @@ public PartitionChunk apply(ShardSpec shardSpec) public void testVersionedIntervalTimelineBehaviorForNumberedShardSpec() { //core partition chunks - PartitionChunk chunk0 = new NumberedShardSpec(0, 2).createChunk("0"); - PartitionChunk chunk1 = new NumberedShardSpec(1, 2).createChunk("1"); + PartitionChunk chunk0 = new NumberedShardSpec(0, 2) + .createChunk(new OvershadowableString("0", 0)); + PartitionChunk chunk1 = new NumberedShardSpec(1, 2) + .createChunk(new OvershadowableString("1", 1)); //appended partition chunk - PartitionChunk chunk4 = new NumberedShardSpec(4, 2).createChunk("4"); + PartitionChunk chunk4 = new NumberedShardSpec(4, 2) + .createChunk(new OvershadowableString("4", 4)); //incomplete partition sets testVersionedIntervalTimelineBehaviorForNumberedShardSpec( @@ -150,53 +156,142 @@ public void testVersionedIntervalTimelineBehaviorForNumberedShardSpec() //complete partition sets testVersionedIntervalTimelineBehaviorForNumberedShardSpec( ImmutableList.of(chunk1, chunk0), - ImmutableSet.of("0", "1") + ImmutableSet.of(new OvershadowableString("0", 0), new OvershadowableString("1", 1)) ); testVersionedIntervalTimelineBehaviorForNumberedShardSpec( ImmutableList.of(chunk4, chunk1, chunk0), - ImmutableSet.of("0", "1", "4") + ImmutableSet.of( + new OvershadowableString("0", 0), + new OvershadowableString("1", 1), + new OvershadowableString("4", 4) + ) ); // a partition set with 0 core partitions - chunk0 = new NumberedShardSpec(0, 0).createChunk("0"); - chunk4 = new NumberedShardSpec(4, 0).createChunk("4"); + chunk0 = new NumberedShardSpec(0, 0).createChunk(new OvershadowableString("0", 0)); + chunk4 = new NumberedShardSpec(4, 0).createChunk(new OvershadowableString("4", 4)); testVersionedIntervalTimelineBehaviorForNumberedShardSpec( ImmutableList.of(chunk0), - ImmutableSet.of("0") + ImmutableSet.of(new OvershadowableString("0", 0)) ); testVersionedIntervalTimelineBehaviorForNumberedShardSpec( ImmutableList.of(chunk4), - ImmutableSet.of("4") + ImmutableSet.of(new OvershadowableString("4", 4)) ); testVersionedIntervalTimelineBehaviorForNumberedShardSpec( ImmutableList.of(chunk4, chunk0), - ImmutableSet.of("0", "4") + ImmutableSet.of(new OvershadowableString("0", 0), new OvershadowableString("4", 4)) ); } private void testVersionedIntervalTimelineBehaviorForNumberedShardSpec( - List> chunks, - Set expectedObjects + List> chunks, + Set expectedObjects ) { - VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(Ordering.natural()); + VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(Ordering.natural()); Interval interval = Intervals.of("2000/3000"); String version = "v1"; - for (PartitionChunk chunk : chunks) { + for (PartitionChunk chunk : chunks) { timeline.add(interval, version, chunk); } - Set actualObjects = new HashSet<>(); - List> entries = timeline.lookup(interval); - for (TimelineObjectHolder entry : entries) { - for (PartitionChunk chunk : entry.getObject()) { + Set actualObjects = new HashSet<>(); + List> entries = timeline.lookup(interval); + for (TimelineObjectHolder entry : entries) { + for (PartitionChunk chunk : entry.getObject()) { actualObjects.add(chunk.getObject()); } } Assert.assertEquals(expectedObjects, actualObjects); } + + @Test + public void testCompatible() + { + final NumberedShardSpec spec = new NumberedShardSpec(0, 0); + Assert.assertTrue(spec.isCompatible(NumberedShardSpec.class)); + Assert.assertTrue(spec.isCompatible(NumberedOverwriteShardSpec.class)); + } + + private static final class OvershadowableString implements Overshadowable + { + private final int partitionId; + private final String val; + + OvershadowableString(String val, int partitionId) + { + this.val = val; + this.partitionId = partitionId; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OvershadowableString that = (OvershadowableString) o; + return partitionId == that.partitionId && + Objects.equals(val, that.val); + } + + @Override + public int hashCode() + { + return Objects.hash(partitionId, val); + } + + @Override + public String toString() + { + return "OvershadowableString{" + + "partitionId=" + partitionId + + ", val='" + val + '\'' + + '}'; + } + + @Override + public boolean overshadows(OvershadowableString other) + { + return false; + } + + @Override + public int getStartRootPartitionId() + { + return partitionId; + } + + @Override + public int getEndRootPartitionId() + { + return partitionId + 1; + } + + @Override + public String getVersion() + { + return "v1"; + } + + @Override + public short getMinorVersion() + { + return 0; + } + + @Override + public short getAtomicUpdateGroupSize() + { + return 1; + } + } } diff --git a/server/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/server/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java index 61e58409a428..86b622a60ee1 100644 --- a/server/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java +++ b/server/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java @@ -132,7 +132,6 @@ public void testIsInChunk() specs.add(new HashOverridenShardSpec(i, 3)); } - assertExistsInOneSpec(specs, new HashInputRow(Integer.MIN_VALUE)); assertExistsInOneSpec(specs, new HashInputRow(Integer.MAX_VALUE)); assertExistsInOneSpec(specs, new HashInputRow(0)); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index ff98ef2517c3..7bfb10e610c0 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -271,8 +271,7 @@ public Authorizer getAuthorizer(String name) ImmutableList.of("met1", "met2"), null, 1, - 53000L, - DataSegment.PruneLoadSpecHolder.DEFAULT + 53000L ); private final DataSegment publishedSegment2 = new DataSegment( "wikipedia2", @@ -283,8 +282,7 @@ public Authorizer getAuthorizer(String name) ImmutableList.of("met1", "met2"), null, 1, - 83000L, - DataSegment.PruneLoadSpecHolder.DEFAULT + 83000L ); private final DataSegment publishedSegment3 = new DataSegment( "wikipedia3", @@ -295,8 +293,7 @@ public Authorizer getAuthorizer(String name) ImmutableList.of("met1", "met2"), null, 1, - 47000L, - DataSegment.PruneLoadSpecHolder.DEFAULT + 47000L ); private final DataSegment segment1 = new DataSegment( @@ -308,8 +305,7 @@ public Authorizer getAuthorizer(String name) ImmutableList.of("met1", "met2"), null, 1, - 100L, - DataSegment.PruneLoadSpecHolder.DEFAULT + 100L ); private final DataSegment segment2 = new DataSegment( "test2", @@ -320,8 +316,7 @@ public Authorizer getAuthorizer(String name) ImmutableList.of("met1", "met2"), null, 1, - 100L, - DataSegment.PruneLoadSpecHolder.DEFAULT + 100L ); private final DataSegment segment3 = new DataSegment( "test3", @@ -332,8 +327,7 @@ public Authorizer getAuthorizer(String name) ImmutableList.of("met1", "met2"), new NumberedShardSpec(2, 3), 1, - 100L, - DataSegment.PruneLoadSpecHolder.DEFAULT + 100L ); private final DataSegment segment4 = new DataSegment( "test4", @@ -344,8 +338,7 @@ public Authorizer getAuthorizer(String name) ImmutableList.of("met1", "met2"), null, 1, - 100L, - DataSegment.PruneLoadSpecHolder.DEFAULT + 100L ); private final DataSegment segment5 = new DataSegment( "test5", @@ -356,8 +349,7 @@ public Authorizer getAuthorizer(String name) ImmutableList.of("met1", "met2"), null, 1, - 100L, - DataSegment.PruneLoadSpecHolder.DEFAULT + 100L ); final List realtimeSegments = ImmutableList.of(segment2, segment4, segment5); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java index 6aac2ce2eab1..30858d68945b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java @@ -47,6 +47,7 @@ import org.apache.druid.query.spec.SpecificSegmentSpec; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.QueryableIndexSegment; +import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.VersionedIntervalTimeline; @@ -63,7 +64,7 @@ public class SpecificSegmentsQuerySegmentWalker implements QuerySegmentWalker, Closeable { private final QueryRunnerFactoryConglomerate conglomerate; - private final Map> timelines = new HashMap<>(); + private final Map> timelines = new HashMap<>(); private final List closeables = new ArrayList<>(); private final List segments = new ArrayList<>(); @@ -78,9 +79,13 @@ public SpecificSegmentsQuerySegmentWalker add( ) { final Segment segment = new QueryableIndexSegment(index, descriptor.getId()); - final VersionedIntervalTimeline timeline = timelines - .computeIfAbsent(descriptor.getDataSource(), dsName -> new VersionedIntervalTimeline<>(Ordering.natural())); - timeline.add(descriptor.getInterval(), descriptor.getVersion(), descriptor.getShardSpec().createChunk(segment)); + final VersionedIntervalTimeline timeline = timelines + .computeIfAbsent(descriptor.getDataSource(), datasource -> new VersionedIntervalTimeline<>(Ordering.natural())); + timeline.add( + descriptor.getInterval(), + descriptor.getVersion(), + descriptor.getShardSpec().createChunk(new ReferenceCountingSegment(segment)) + ); segments.add(descriptor); closeables.add(index); return this; @@ -132,7 +137,7 @@ public QueryRunner getQueryRunnerForIntervals( )) .build(); } - final VersionedIntervalTimeline timeline = getTimelineForTableDataSource( + final VersionedIntervalTimeline timeline = getTimelineForTableDataSource( newQuery1); return makeBaseRunner( newQuery1, @@ -196,7 +201,7 @@ public void close() throws IOException } } - private VersionedIntervalTimeline getTimelineForTableDataSource(Query query) + private VersionedIntervalTimeline getTimelineForTableDataSource(Query query) { if (query.getDataSource() instanceof TableDataSource) { return timelines.get(((TableDataSource) query.getDataSource()).getName()); @@ -212,7 +217,7 @@ private QueryRunner makeBaseRunner( final Iterable specs ) { - final VersionedIntervalTimeline timeline = getTimelineForTableDataSource(query); + final VersionedIntervalTimeline timeline = getTimelineForTableDataSource(query); if (timeline == null) { return new NoopQueryRunner<>(); } @@ -225,7 +230,7 @@ private QueryRunner makeBaseRunner( .create(specs) .transformCat( descriptor -> { - final PartitionHolder holder = timeline.findEntry( + final PartitionHolder holder = timeline.findEntry( descriptor.getInterval(), descriptor.getVersion() );