diff --git a/core/src/main/java/org/apache/druid/segment/column/TypeStrategies.java b/core/src/main/java/org/apache/druid/segment/column/TypeStrategies.java index 54a15b1dbdf3..20507c597b0f 100644 --- a/core/src/main/java/org/apache/druid/segment/column/TypeStrategies.java +++ b/core/src/main/java/org/apache/druid/segment/column/TypeStrategies.java @@ -236,8 +236,6 @@ public static void checkMaxSize(int available, int maxSizeBytes, TypeSignature { - private static final Comparator COMPARATOR = Longs::compare; - @Override public int estimateSizeBytes(Long value) { @@ -276,9 +274,9 @@ public int write(ByteBuffer buffer, Long value, int maxSizeBytes) } @Override - public int compare(Long o1, Long o2) + public int compare(Object o1, Object o2) { - return COMPARATOR.compare(o1, o2); + return Longs.compare(((Number) o1).longValue(), ((Number) o2).longValue()); } } @@ -289,8 +287,6 @@ public int compare(Long o1, Long o2) */ public static final class FloatTypeStrategy implements TypeStrategy { - private static final Comparator COMPARATOR = Floats::compare; - @Override public int estimateSizeBytes(Float value) { @@ -329,9 +325,9 @@ public int write(ByteBuffer buffer, Float value, int maxSizeBytes) } @Override - public int compare(Float o1, Float o2) + public int compare(Object o1, Object o2) { - return COMPARATOR.compare(o1, o2); + return Floats.compare(((Number) o1).floatValue(), ((Number) o2).floatValue()); } } @@ -342,7 +338,6 @@ public int compare(Float o1, Float o2) */ public static final class DoubleTypeStrategy implements TypeStrategy { - private static final Comparator COMPARATOR = Double::compare; @Override public int estimateSizeBytes(Double value) @@ -382,9 +377,9 @@ public int write(ByteBuffer buffer, Double value, int maxSizeBytes) } @Override - public int compare(Double o1, Double o2) + public int compare(Object o1, Object o2) { - return COMPARATOR.compare(o1, o2); + return Double.compare(((Number) o1).doubleValue(), ((Number) o2).doubleValue()); } } @@ -437,7 +432,7 @@ public int write(ByteBuffer buffer, String value, int maxSizeBytes) } @Override - public int compare(String s, String s2) + public int compare(Object s, Object s2) { // copy of lexicographical string comparator in druid processing // Avoid comparisons for equal references @@ -447,7 +442,7 @@ public int compare(String s, String s2) return 0; } - return ORDERING.compare(s, s2); + return ORDERING.compare((String) s, (String) s2); } } @@ -521,8 +516,11 @@ public int write(ByteBuffer buffer, Object[] value, int maxSizeBytes) } @Override - public int compare(@Nullable Object[] o1, @Nullable Object[] o2) + public int compare(@Nullable Object o1Obj, @Nullable Object o2Obj) { + Object[] o1 = (Object[]) o1Obj; + Object[] o2 = (Object[]) o2Obj; + //noinspection ArrayEquality if (o1 == o2) { return 0; diff --git a/core/src/main/java/org/apache/druid/segment/column/TypeStrategy.java b/core/src/main/java/org/apache/druid/segment/column/TypeStrategy.java index 8a97882d54df..e4856f889714 100644 --- a/core/src/main/java/org/apache/druid/segment/column/TypeStrategy.java +++ b/core/src/main/java/org/apache/druid/segment/column/TypeStrategy.java @@ -55,8 +55,15 @@ * Implementations of this interface should be thread safe, but may not use {@link ByteBuffer} in a thread safe manner, * potentially modifying positions and limits, either temporarily or permanently depending on which set of methods is * called. + * + * This interface extends {@code Comparator} instead of {@code Comparator} because trying to specialize the + * type of the comparison method can run into issues for comparators of objects that can sometimes be of a different + * java class type. For example, {@code Comparator} cannot accept Integer objects in its comparison method + * and there is no easy way for this interface definition to allow {@code TypeStrategy} to actually be a + * {@code Comparator}. So, we fall back to effectively erasing the generic type and having them all be + * {@code Comparator}. */ -public interface TypeStrategy extends Comparator +public interface TypeStrategy extends Comparator { /** * Estimate the size in bytes that writing this value to memory would require. This method is not required to be diff --git a/core/src/test/java/org/apache/druid/segment/column/TypeStrategiesTest.java b/core/src/test/java/org/apache/druid/segment/column/TypeStrategiesTest.java index fa6d86d21f50..66f4adcdd02d 100644 --- a/core/src/test/java/org/apache/druid/segment/column/TypeStrategiesTest.java +++ b/core/src/test/java/org/apache/druid/segment/column/TypeStrategiesTest.java @@ -19,6 +19,7 @@ package org.apache.druid.segment.column; +import com.google.common.collect.Ordering; import com.google.common.primitives.Longs; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Pair; @@ -103,7 +104,7 @@ public int write(ByteBuffer buffer, String value, int maxSizeBytes) } @Override - public int compare(String o1, String o2) + public int compare(Object o1, Object o2) { return 0; } @@ -639,10 +640,13 @@ public int compareTo(NullableLongPair o) public static class NullableLongPairTypeStrategy implements TypeStrategy { + + private Ordering ordering = Comparators.naturalNullsFirst(); + @Override - public int compare(NullableLongPair o1, NullableLongPair o2) + public int compare(Object o1, Object o2) { - return Comparators.naturalNullsFirst().compare(o1, o2); + return ordering.compare((NullableLongPair) o1, (NullableLongPair) o2); } @Override diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/ComplexFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/ComplexFrameColumnReader.java index f7b662d42bb2..e4a39c16f753 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/ComplexFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/ComplexFrameColumnReader.java @@ -22,7 +22,7 @@ import com.google.common.primitives.Ints; import org.apache.datasketches.memory.Memory; import org.apache.druid.frame.Frame; -import org.apache.druid.frame.write.columnar.ComplexFrameColumnWriter; +import org.apache.druid.frame.write.columnar.ComplexFrameMaker; import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -53,10 +53,10 @@ public ColumnPlus readColumn(final Frame frame) final Memory memory = frame.region(columnNumber); validate(memory, frame.numRows()); - final int typeNameLength = memory.getInt(ComplexFrameColumnWriter.TYPE_NAME_LENGTH_POSITION); + final int typeNameLength = memory.getInt(ComplexFrameMaker.TYPE_NAME_LENGTH_POSITION); final byte[] typeNameBytes = new byte[typeNameLength]; - memory.getByteArray(ComplexFrameColumnWriter.TYPE_NAME_POSITION, typeNameBytes, 0, typeNameLength); + memory.getByteArray(ComplexFrameMaker.TYPE_NAME_POSITION, typeNameBytes, 0, typeNameLength); final String typeName = StringUtils.fromUtf8(typeNameBytes); final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName); @@ -84,7 +84,7 @@ public ColumnPlus readColumn(final Frame frame) private void validate(final Memory region, final int numRows) { - if (region.getCapacity() < ComplexFrameColumnWriter.TYPE_NAME_POSITION) { + if (region.getCapacity() < ComplexFrameMaker.TYPE_NAME_POSITION) { throw new ISE("Column is not big enough for a header"); } @@ -93,9 +93,9 @@ private void validate(final Memory region, final int numRows) throw new ISE("Column does not have the correct type code"); } - final int typeNameLength = region.getInt(ComplexFrameColumnWriter.TYPE_NAME_LENGTH_POSITION); + final int typeNameLength = region.getInt(ComplexFrameMaker.TYPE_NAME_LENGTH_POSITION); if (region.getCapacity() < - ComplexFrameColumnWriter.TYPE_NAME_POSITION + typeNameLength + (long) numRows * Integer.BYTES) { + ComplexFrameMaker.TYPE_NAME_POSITION + typeNameLength + (long) numRows * Integer.BYTES) { throw new ISE("Column is missing offset section"); } } @@ -198,7 +198,7 @@ private Object getObjectForPhysicalRow(final int physicalRow) startOfDataSection + memory.getInt(startOfOffsetSection + (long) Integer.BYTES * (physicalRow - 1)); } - if (memory.getByte(startOffset) == ComplexFrameColumnWriter.NULL_MARKER) { + if (memory.getByte(startOffset) == ComplexFrameMaker.NULL_MARKER) { return null; } else { final int payloadLength = Ints.checkedCast(endOffset - startOffset - Byte.BYTES); diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleFrameColumnReader.java index a8f08feaa6a4..8ac4b1d15d65 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleFrameColumnReader.java @@ -22,7 +22,7 @@ import org.apache.datasketches.memory.Memory; import org.apache.druid.common.config.NullHandling; import org.apache.druid.frame.Frame; -import org.apache.druid.frame.write.columnar.DoubleFrameColumnWriter; +import org.apache.druid.frame.write.columnar.DoubleFrameMaker; import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.java.util.common.ISE; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -69,7 +69,7 @@ private void validate(final Memory region, final int numRows) final long memorySize = region.getCapacity(); // Check if column is big enough for a header - if (memorySize < DoubleFrameColumnWriter.DATA_OFFSET) { + if (memorySize < DoubleFrameMaker.DATA_OFFSET) { throw new ISE("Column is not big enough for a header"); } @@ -79,10 +79,10 @@ private void validate(final Memory region, final int numRows) } final boolean hasNulls = getHasNulls(region); - final int sz = DoubleFrameColumnWriter.valueSize(hasNulls); + final int sz = DoubleFrameMaker.valueSize(hasNulls); // Check column length again, now that we know exactly how long it should be. - if (memorySize != DoubleFrameColumnWriter.DATA_OFFSET + (long) sz * numRows) { + if (memorySize != DoubleFrameMaker.DATA_OFFSET + (long) sz * numRows) { throw new ISE("Column does not have the correct length"); } } @@ -108,9 +108,9 @@ private DoubleFrameColumn( { this.frame = frame; this.hasNulls = hasNulls; - this.sz = DoubleFrameColumnWriter.valueSize(hasNulls); + this.sz = DoubleFrameMaker.valueSize(hasNulls); this.memory = memory; - this.memoryPosition = DoubleFrameColumnWriter.DATA_OFFSET; + this.memoryPosition = DoubleFrameMaker.DATA_OFFSET; } @Override diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatFrameColumnReader.java index dc5c7fa18f27..0f1d09f57bc5 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatFrameColumnReader.java @@ -22,7 +22,7 @@ import org.apache.datasketches.memory.Memory; import org.apache.druid.common.config.NullHandling; import org.apache.druid.frame.Frame; -import org.apache.druid.frame.write.columnar.FloatFrameColumnWriter; +import org.apache.druid.frame.write.columnar.FloatFrameMaker; import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.java.util.common.ISE; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -69,7 +69,7 @@ private void validate(final Memory region, final int numRows) final long memorySize = region.getCapacity(); // Check if column is big enough for a header - if (memorySize < FloatFrameColumnWriter.DATA_OFFSET) { + if (memorySize < FloatFrameMaker.DATA_OFFSET) { throw new ISE("Column is not big enough for a header"); } @@ -79,10 +79,10 @@ private void validate(final Memory region, final int numRows) } final boolean hasNulls = getHasNulls(region); - final int sz = FloatFrameColumnWriter.valueSize(hasNulls); + final int sz = FloatFrameMaker.valueSize(hasNulls); // Check column length again, now that we know exactly how long it should be. - if (memorySize != FloatFrameColumnWriter.DATA_OFFSET + (long) sz * numRows) { + if (memorySize != FloatFrameMaker.DATA_OFFSET + (long) sz * numRows) { throw new ISE("Column does not have the correct length"); } } @@ -108,9 +108,9 @@ private FloatFrameColumn( { this.frame = frame; this.hasNulls = hasNulls; - this.sz = FloatFrameColumnWriter.valueSize(hasNulls); + this.sz = FloatFrameMaker.valueSize(hasNulls); this.memory = memory; - this.memoryPosition = FloatFrameColumnWriter.DATA_OFFSET; + this.memoryPosition = FloatFrameMaker.DATA_OFFSET; } @Override diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongFrameColumnReader.java index 91bd898597bf..1662411b912b 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongFrameColumnReader.java @@ -23,7 +23,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.frame.Frame; import org.apache.druid.frame.write.columnar.FrameColumnWriters; -import org.apache.druid.frame.write.columnar.LongFrameColumnWriter; +import org.apache.druid.frame.write.columnar.LongFrameMaker; import org.apache.druid.java.util.common.ISE; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.ColumnValueSelector; @@ -67,7 +67,7 @@ public ColumnPlus readColumn(final Frame frame) private void validate(final Memory region, final int numRows) { // Check if column is big enough for a header - if (region.getCapacity() < LongFrameColumnWriter.DATA_OFFSET) { + if (region.getCapacity() < LongFrameMaker.DATA_OFFSET) { throw new ISE("Column is not big enough for a header"); } @@ -77,10 +77,10 @@ private void validate(final Memory region, final int numRows) } final boolean hasNulls = getHasNulls(region); - final int sz = LongFrameColumnWriter.valueSize(hasNulls); + final int sz = LongFrameMaker.valueSize(hasNulls); // Check column length again, now that we know exactly how long it should be. - if (region.getCapacity() != LongFrameColumnWriter.DATA_OFFSET + (long) sz * numRows) { + if (region.getCapacity() != LongFrameMaker.DATA_OFFSET + (long) sz * numRows) { throw new ISE("Column does not have the correct length"); } } @@ -106,9 +106,9 @@ private LongFrameColumn( { this.frame = frame; this.hasNulls = hasNulls; - this.sz = LongFrameColumnWriter.valueSize(hasNulls); + this.sz = LongFrameMaker.valueSize(hasNulls); this.memory = memory; - this.memoryPosition = LongFrameColumnWriter.DATA_OFFSET; + this.memoryPosition = LongFrameMaker.DATA_OFFSET; } @Override diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/ComplexFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/ComplexFrameColumnWriter.java index 08a8aa0b67b4..6e6b985334df 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/ComplexFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/ComplexFrameColumnWriter.java @@ -19,14 +19,9 @@ package org.apache.druid.frame.write.columnar; -import com.google.common.primitives.Ints; -import it.unimi.dsi.fastutil.bytes.ByteArrays; import org.apache.datasketches.memory.WritableMemory; -import org.apache.druid.frame.allocation.AppendableMemory; import org.apache.druid.frame.allocation.MemoryAllocator; -import org.apache.druid.frame.allocation.MemoryRange; import org.apache.druid.frame.read.columnar.ComplexFrameColumnReader; -import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.segment.BaseObjectColumnValueSelector; import org.apache.druid.segment.serde.ComplexMetricSerde; @@ -38,22 +33,9 @@ */ public class ComplexFrameColumnWriter implements FrameColumnWriter { - // Less than half of AppendableMemory.DEFAULT_INITIAL_ALLOCATION_SIZE. - // This guarantees we can fit a WorkerMemoryParmeters.MAX_FRAME_COLUMNS number of columns into a frame. - private static final int INITIAL_ALLOCATION_SIZE = 128; - - public static final byte NOT_NULL_MARKER = 0x00; - public static final byte NULL_MARKER = 0x01; - public static final int TYPE_NAME_LENGTH_POSITION = Byte.BYTES; - public static final int TYPE_NAME_POSITION = Byte.BYTES + Integer.BYTES; - private final ComplexMetricSerde serde; private final BaseObjectColumnValueSelector selector; - private final AppendableMemory offsetMemory; - private final AppendableMemory dataMemory; - private final byte[] typeNameBytes; - - private int lastDataLength = -1; + private final ComplexFrameMaker maker; ComplexFrameColumnWriter( final BaseObjectColumnValueSelector selector, @@ -63,95 +45,37 @@ public class ComplexFrameColumnWriter implements FrameColumnWriter { this.selector = selector; this.serde = serde; - this.offsetMemory = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); - this.dataMemory = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); - this.typeNameBytes = StringUtils.toUtf8(serde.getTypeName()); + this.maker = new ComplexFrameMaker(allocator, StringUtils.toUtf8(serde.getTypeName())); } @Override public boolean addSelection() { - if (!offsetMemory.reserveAdditional(Integer.BYTES)) { - return false; - } - final Object complexObject = selector.getObject(); - final byte[] complexBytes = complexObject == null ? ByteArrays.EMPTY_ARRAY : serde.toBytes(complexObject); - - if (complexBytes.length == Integer.MAX_VALUE) { - // Cannot handle objects this large. - return false; - } - - final int dataLength = complexBytes.length + 1; - - if (dataMemory.size() + dataLength > Integer.MAX_VALUE || !(dataMemory.reserveAdditional(dataLength))) { - return false; - } - - // All space is reserved. Start writing. - final MemoryRange offsetCursor = offsetMemory.cursor(); - offsetCursor.memory().putInt(offsetCursor.start(), Ints.checkedCast(dataMemory.size() + dataLength)); - offsetMemory.advanceCursor(Integer.BYTES); - - final MemoryRange dataCursor = dataMemory.cursor(); - dataCursor.memory().putByte(dataCursor.start(), complexObject == null ? NULL_MARKER : NOT_NULL_MARKER); - dataCursor.memory().putByteArray(dataCursor.start() + 1, complexBytes, 0, complexBytes.length); - dataMemory.advanceCursor(dataLength); - - lastDataLength = dataLength; - return true; + return maker.add(complexObject == null ? null : serde.toBytes(complexObject)); } @Override public void undo() { - if (lastDataLength == -1) { - throw new ISE("Nothing to undo"); - } - - offsetMemory.rewindCursor(Integer.BYTES); - dataMemory.rewindCursor(lastDataLength); - lastDataLength = -1; + maker.undo(); } @Override public long size() { - return headerSize() + offsetMemory.size() + dataMemory.size(); + return maker.size(); } @Override public long writeTo(final WritableMemory memory, final long startPosition) { - long currentPosition = startPosition; - - memory.putByte(currentPosition, FrameColumnWriters.TYPE_COMPLEX); - currentPosition += 1; - - memory.putInt(currentPosition, typeNameBytes.length); - currentPosition += Integer.BYTES; - - memory.putByteArray(currentPosition, typeNameBytes, 0, typeNameBytes.length); - currentPosition += typeNameBytes.length; - - currentPosition += offsetMemory.writeTo(memory, currentPosition); - currentPosition += dataMemory.writeTo(memory, currentPosition); - - return currentPosition - startPosition; + return maker.writeTo(memory, startPosition); } @Override public void close() { - offsetMemory.close(); - dataMemory.close(); - } - - private int headerSize() - { - return 1 /* type code */ - + Integer.BYTES /* type name length */ - + typeNameBytes.length; + maker.close(); } } diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/ComplexFrameMaker.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/ComplexFrameMaker.java new file mode 100644 index 000000000000..8622a7bce202 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/ComplexFrameMaker.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.write.columnar; + +import com.google.common.primitives.Ints; +import it.unimi.dsi.fastutil.bytes.ByteArrays; +import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.frame.allocation.AppendableMemory; +import org.apache.druid.frame.allocation.MemoryAllocator; +import org.apache.druid.frame.allocation.MemoryRange; +import org.apache.druid.frame.read.columnar.ComplexFrameColumnReader; +import org.apache.druid.java.util.common.ISE; + +/** + * Column writer for complex columns. + * + * Dual to {@link ComplexFrameColumnReader}. + */ +public class ComplexFrameMaker +{ + // Less than half of AppendableMemory.DEFAULT_INITIAL_ALLOCATION_SIZE. + // This guarantees we can fit a WorkerMemoryParmeters.MAX_FRAME_COLUMNS number of columns into a frame. + private static final int INITIAL_ALLOCATION_SIZE = 128; + + public static final byte NOT_NULL_MARKER = 0x00; + public static final byte NULL_MARKER = 0x01; + public static final int TYPE_NAME_LENGTH_POSITION = Byte.BYTES; + public static final int TYPE_NAME_POSITION = Byte.BYTES + Integer.BYTES; + + private final AppendableMemory offsetMemory; + private final AppendableMemory dataMemory; + private final byte[] typeName; + + private int lastDataLength = -1; + + ComplexFrameMaker( + final MemoryAllocator allocator, + final byte[] typeName + ) + { + this.offsetMemory = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); + this.dataMemory = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); + this.typeName = typeName; + } + + public boolean add(byte[] bytes) + { + if (!offsetMemory.reserveAdditional(Integer.BYTES)) { + return false; + } + + boolean isNull = false; + if (bytes == null) { + isNull = true; + bytes = ByteArrays.EMPTY_ARRAY; + } + + if (bytes.length == Integer.MAX_VALUE) { + // Cannot handle objects this large. + return false; + } + + final int dataLength = bytes.length + 1; + + if (dataMemory.size() + dataLength > Integer.MAX_VALUE || !(dataMemory.reserveAdditional(dataLength))) { + return false; + } + + // All space is reserved. Start writing. + final MemoryRange offsetCursor = offsetMemory.cursor(); + offsetCursor.memory().putInt(offsetCursor.start(), Ints.checkedCast(dataMemory.size() + dataLength)); + offsetMemory.advanceCursor(Integer.BYTES); + + final MemoryRange dataCursor = dataMemory.cursor(); + dataCursor.memory().putByte(dataCursor.start(), isNull ? NULL_MARKER : NOT_NULL_MARKER); + dataCursor.memory().putByteArray(dataCursor.start() + 1, bytes, 0, bytes.length); + dataMemory.advanceCursor(dataLength); + + lastDataLength = dataLength; + return true; + } + + public void undo() + { + if (lastDataLength == -1) { + throw new ISE("Nothing to undo"); + } + + offsetMemory.rewindCursor(Integer.BYTES); + dataMemory.rewindCursor(lastDataLength); + lastDataLength = -1; + } + + public long size() + { + return headerSize() + offsetMemory.size() + dataMemory.size(); + } + + public long writeTo(final WritableMemory memory, final long startPosition) + { + long currentPosition = startPosition; + + memory.putByte(currentPosition, FrameColumnWriters.TYPE_COMPLEX); + currentPosition += 1; + + memory.putInt(currentPosition, typeName.length); + currentPosition += Integer.BYTES; + + memory.putByteArray(currentPosition, typeName, 0, typeName.length); + currentPosition += typeName.length; + + currentPosition += offsetMemory.writeTo(memory, currentPosition); + currentPosition += dataMemory.writeTo(memory, currentPosition); + + return currentPosition - startPosition; + } + + public void close() + { + offsetMemory.close(); + dataMemory.close(); + } + + private int headerSize() + { + return 1 /* type code */ + + Integer.BYTES /* type name length */ + + typeName.length; + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleFrameColumnWriter.java index 82d2054d4029..f83e210ef06d 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleFrameColumnWriter.java @@ -20,19 +20,13 @@ package org.apache.druid.frame.write.columnar; import org.apache.datasketches.memory.WritableMemory; -import org.apache.druid.frame.allocation.AppendableMemory; import org.apache.druid.frame.allocation.MemoryAllocator; -import org.apache.druid.frame.allocation.MemoryRange; import org.apache.druid.segment.BaseDoubleColumnValueSelector; public class DoubleFrameColumnWriter implements FrameColumnWriter { - public static final long DATA_OFFSET = 1 /* type code */ + 1 /* has nulls? */; - private final BaseDoubleColumnValueSelector selector; - private final AppendableMemory appendableMemory; - private final boolean hasNulls; - private final int sz; + private final DoubleFrameMaker maker; DoubleFrameColumnWriter( BaseDoubleColumnValueSelector selector, @@ -41,71 +35,40 @@ public class DoubleFrameColumnWriter implements FrameColumnWriter ) { this.selector = selector; - this.appendableMemory = AppendableMemory.create(allocator); - this.hasNulls = hasNulls; - this.sz = valueSize(hasNulls); - } - - public static int valueSize(final boolean hasNulls) - { - return hasNulls ? Double.BYTES + 1 : Double.BYTES; + this.maker = new DoubleFrameMaker(allocator, hasNulls); } @Override public boolean addSelection() { - if (!(appendableMemory.reserveAdditional(sz))) { - return false; - } - - final MemoryRange cursor = appendableMemory.cursor(); - final WritableMemory memory = cursor.memory(); - final long position = cursor.start(); - - if (hasNulls) { - if (selector.isNull()) { - memory.putByte(position, (byte) 1); - memory.putDouble(position + 1, 0); - } else { - memory.putByte(position, (byte) 0); - memory.putDouble(position + 1, selector.getDouble()); - } + if (selector.isNull()) { + return maker.addNull(); } else { - memory.putDouble(position, selector.getDouble()); + return maker.add(selector.getDouble()); } - - appendableMemory.advanceCursor(sz); - return true; } @Override public void undo() { - appendableMemory.rewindCursor(sz); + maker.undo(); } @Override public long size() { - return DATA_OFFSET + appendableMemory.size(); + return maker.size(); } @Override public long writeTo(final WritableMemory memory, final long startPosition) { - long currentPosition = startPosition; - - memory.putByte(currentPosition, FrameColumnWriters.TYPE_DOUBLE); - memory.putByte(currentPosition + 1, hasNulls ? (byte) 1 : (byte) 0); - currentPosition += 2; - - currentPosition += appendableMemory.writeTo(memory, currentPosition); - return currentPosition - startPosition; + return maker.writeTo(memory, startPosition); } @Override public void close() { - appendableMemory.close(); + maker.close(); } } diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleFrameMaker.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleFrameMaker.java new file mode 100644 index 000000000000..13ee53d6d5fc --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleFrameMaker.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.write.columnar; + +import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.frame.allocation.AppendableMemory; +import org.apache.druid.frame.allocation.MemoryAllocator; +import org.apache.druid.frame.allocation.MemoryRange; +import org.apache.druid.java.util.common.ISE; + +import java.io.Closeable; + +public class DoubleFrameMaker implements Closeable +{ + public static final long DATA_OFFSET = 1 /* type code */ + 1 /* has nulls? */; + + private final AppendableMemory appendableMemory; + private final boolean hasNulls; + private final int sz; + + public DoubleFrameMaker( + MemoryAllocator allocator, + boolean hasNulls + ) + { + this.appendableMemory = AppendableMemory.create(allocator); + this.hasNulls = hasNulls; + this.sz = valueSize(hasNulls); + } + + public static int valueSize(final boolean hasNulls) + { + return hasNulls ? Double.BYTES + 1 : Double.BYTES; + } + + public boolean add(double value) + { + if (!(appendableMemory.reserveAdditional(sz))) { + return false; + } + + final MemoryRange cursor = appendableMemory.cursor(); + final WritableMemory memory = cursor.memory(); + final long position = cursor.start(); + + if (hasNulls) { + memory.putByte(position, (byte) 0); + memory.putDouble(position + 1, value); + } else { + memory.putDouble(position, value); + } + + appendableMemory.advanceCursor(sz); + return true; + } + + public boolean addNull() + { + if (!hasNulls) { + throw new ISE("Was told that null doesn't exist, cannot add null"); + } + + if (!(appendableMemory.reserveAdditional(sz))) { + return false; + } + + final MemoryRange cursor = appendableMemory.cursor(); + final WritableMemory memory = cursor.memory(); + final long position = cursor.start(); + + memory.putByte(position, (byte) 1); + memory.putDouble(position + 1, 0); + + appendableMemory.advanceCursor(sz); + return true; + } + + public void undo() + { + appendableMemory.rewindCursor(sz); + } + + public long size() + { + return DATA_OFFSET + appendableMemory.size(); + } + + public long writeTo(final WritableMemory memory, final long startPosition) + { + long currentPosition = startPosition; + + memory.putByte(currentPosition, FrameColumnWriters.TYPE_DOUBLE); + memory.putByte(currentPosition + 1, hasNulls ? (byte) 1 : (byte) 0); + currentPosition += 2; + + currentPosition += appendableMemory.writeTo(memory, currentPosition); + return currentPosition - startPosition; + } + + @Override + public void close() + { + appendableMemory.close(); + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatFrameColumnWriter.java index 900b72d2076c..5f7843483d5a 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatFrameColumnWriter.java @@ -20,19 +20,13 @@ package org.apache.druid.frame.write.columnar; import org.apache.datasketches.memory.WritableMemory; -import org.apache.druid.frame.allocation.AppendableMemory; import org.apache.druid.frame.allocation.MemoryAllocator; -import org.apache.druid.frame.allocation.MemoryRange; import org.apache.druid.segment.BaseFloatColumnValueSelector; public class FloatFrameColumnWriter implements FrameColumnWriter { - public static final long DATA_OFFSET = 1 /* type code */ + 1 /* has nulls? */; - private final BaseFloatColumnValueSelector selector; - private final AppendableMemory appendableMemory; - private final boolean hasNulls; - private final int sz; + private final FloatFrameMaker maker; FloatFrameColumnWriter( BaseFloatColumnValueSelector selector, @@ -41,71 +35,40 @@ public class FloatFrameColumnWriter implements FrameColumnWriter ) { this.selector = selector; - this.appendableMemory = AppendableMemory.create(allocator); - this.hasNulls = hasNulls; - this.sz = valueSize(hasNulls); - } - - public static int valueSize(final boolean hasNulls) - { - return hasNulls ? Float.BYTES + 1 : Float.BYTES; + this.maker = new FloatFrameMaker(allocator, hasNulls); } @Override public boolean addSelection() { - if (!(appendableMemory.reserveAdditional(sz))) { - return false; - } - - final MemoryRange cursor = appendableMemory.cursor(); - final WritableMemory memory = cursor.memory(); - final long position = cursor.start(); - - if (hasNulls) { - if (selector.isNull()) { - memory.putByte(position, (byte) 1); - memory.putFloat(position + 1, 0); - } else { - memory.putByte(position, (byte) 0); - memory.putFloat(position + 1, selector.getFloat()); - } + if (selector.isNull()) { + return maker.addNull(); } else { - memory.putFloat(position, selector.getFloat()); + return maker.add(selector.getFloat()); } - - appendableMemory.advanceCursor(sz); - return true; } @Override public void undo() { - appendableMemory.rewindCursor(sz); + maker.undo(); } @Override public long size() { - return DATA_OFFSET + appendableMemory.size(); + return maker.size(); } @Override public long writeTo(final WritableMemory memory, final long startPosition) { - long currentPosition = startPosition; - - memory.putByte(currentPosition, FrameColumnWriters.TYPE_FLOAT); - memory.putByte(currentPosition + 1, hasNulls ? (byte) 1 : (byte) 0); - currentPosition += 2; - - currentPosition += appendableMemory.writeTo(memory, currentPosition); - return currentPosition - startPosition; + return maker.writeTo(memory, startPosition); } @Override public void close() { - appendableMemory.close(); + maker.close(); } } diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatFrameMaker.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatFrameMaker.java new file mode 100644 index 000000000000..3c299d0b9106 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatFrameMaker.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.write.columnar; + +import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.frame.allocation.AppendableMemory; +import org.apache.druid.frame.allocation.MemoryAllocator; +import org.apache.druid.frame.allocation.MemoryRange; +import org.apache.druid.java.util.common.ISE; + +import java.io.Closeable; + +public class FloatFrameMaker implements Closeable +{ + public static final long DATA_OFFSET = 1 /* type code */ + 1 /* has nulls? */; + + private final AppendableMemory appendableMemory; + private final boolean hasNulls; + private final int sz; + + public FloatFrameMaker( + MemoryAllocator allocator, + boolean hasNulls + ) + { + this.appendableMemory = AppendableMemory.create(allocator); + this.hasNulls = hasNulls; + this.sz = valueSize(hasNulls); + } + + public static int valueSize(final boolean hasNulls) + { + return hasNulls ? Float.BYTES + 1 : Float.BYTES; + } + + public boolean add(float value) + { + if (!(appendableMemory.reserveAdditional(sz))) { + return false; + } + + final MemoryRange cursor = appendableMemory.cursor(); + final WritableMemory memory = cursor.memory(); + final long position = cursor.start(); + + if (hasNulls) { + memory.putByte(position, (byte) 0); + memory.putFloat(position + 1, value); + } else { + memory.putFloat(position, value); + } + + appendableMemory.advanceCursor(sz); + return true; + } + + public boolean addNull() + { + if (!hasNulls) { + throw new ISE("Was told that null doesn't exist, cannot add null"); + } + + if (!(appendableMemory.reserveAdditional(sz))) { + return false; + } + + final MemoryRange cursor = appendableMemory.cursor(); + final WritableMemory memory = cursor.memory(); + final long position = cursor.start(); + + memory.putByte(position, (byte) 1); + memory.putFloat(position + 1, 0); + + appendableMemory.advanceCursor(sz); + return true; + } + + public void undo() + { + appendableMemory.rewindCursor(sz); + } + + public long size() + { + return DATA_OFFSET + appendableMemory.size(); + } + + public long writeTo(final WritableMemory memory, final long startPosition) + { + long currentPosition = startPosition; + + memory.putByte(currentPosition, FrameColumnWriters.TYPE_FLOAT); + memory.putByte(currentPosition + 1, hasNulls ? (byte) 1 : (byte) 0); + currentPosition += 2; + + currentPosition += appendableMemory.writeTo(memory, currentPosition); + return currentPosition - startPosition; + } + + @Override + public void close() + { + appendableMemory.close(); + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongFrameColumnWriter.java index c0a2bf4bb3ab..101d36a83a07 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongFrameColumnWriter.java @@ -20,19 +20,13 @@ package org.apache.druid.frame.write.columnar; import org.apache.datasketches.memory.WritableMemory; -import org.apache.druid.frame.allocation.AppendableMemory; import org.apache.druid.frame.allocation.MemoryAllocator; -import org.apache.druid.frame.allocation.MemoryRange; import org.apache.druid.segment.BaseLongColumnValueSelector; public class LongFrameColumnWriter implements FrameColumnWriter { - public static final long DATA_OFFSET = 1 /* type code */ + 1 /* has nulls? */; - private final BaseLongColumnValueSelector selector; - private final AppendableMemory appendableMemory; - private final boolean hasNulls; - private final int sz; + private final LongFrameMaker maker; LongFrameColumnWriter( BaseLongColumnValueSelector selector, @@ -41,71 +35,40 @@ public class LongFrameColumnWriter implements FrameColumnWriter ) { this.selector = selector; - this.appendableMemory = AppendableMemory.create(allocator); - this.hasNulls = hasNulls; - this.sz = valueSize(hasNulls); - } - - public static int valueSize(final boolean hasNulls) - { - return hasNulls ? Long.BYTES + 1 : Long.BYTES; + this.maker = new LongFrameMaker(allocator, hasNulls); } @Override public boolean addSelection() { - if (!(appendableMemory.reserveAdditional(sz))) { - return false; - } - - final MemoryRange cursor = appendableMemory.cursor(); - final WritableMemory memory = cursor.memory(); - final long position = cursor.start(); - - if (hasNulls) { - if (selector.isNull()) { - memory.putByte(position, (byte) 1); - memory.putLong(position + 1, 0); - } else { - memory.putByte(position, (byte) 0); - memory.putLong(position + 1, selector.getLong()); - } + if (selector.isNull()) { + return maker.addNull(); } else { - memory.putLong(position, selector.getLong()); + return maker.add(selector.getLong()); } - - appendableMemory.advanceCursor(sz); - return true; } @Override public void undo() { - appendableMemory.rewindCursor(sz); + maker.undo(); } @Override public long size() { - return DATA_OFFSET + appendableMemory.size(); + return maker.size(); } @Override public long writeTo(final WritableMemory memory, final long startPosition) { - long currentPosition = startPosition; - - memory.putByte(currentPosition, FrameColumnWriters.TYPE_LONG); - memory.putByte(currentPosition + 1, hasNulls ? (byte) 1 : (byte) 0); - currentPosition += 2; - - currentPosition += appendableMemory.writeTo(memory, currentPosition); - return currentPosition - startPosition; + return maker.writeTo(memory, startPosition); } @Override public void close() { - appendableMemory.close(); + maker.close(); } } diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongFrameMaker.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongFrameMaker.java new file mode 100644 index 000000000000..54c4e2bc98de --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongFrameMaker.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.write.columnar; + +import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.frame.allocation.AppendableMemory; +import org.apache.druid.frame.allocation.MemoryAllocator; +import org.apache.druid.frame.allocation.MemoryRange; +import org.apache.druid.java.util.common.ISE; + +import java.io.Closeable; + +public class LongFrameMaker implements Closeable +{ + public static final long DATA_OFFSET = 1 /* type code */ + 1 /* has nulls? */; + + private final AppendableMemory appendableMemory; + private final boolean hasNulls; + private final int sz; + + public LongFrameMaker( + MemoryAllocator allocator, + boolean hasNulls + ) + { + this.appendableMemory = AppendableMemory.create(allocator); + this.hasNulls = hasNulls; + this.sz = valueSize(hasNulls); + } + + public static int valueSize(final boolean hasNulls) + { + return hasNulls ? Long.BYTES + 1 : Long.BYTES; + } + + public boolean add(long value) + { + if (!(appendableMemory.reserveAdditional(sz))) { + return false; + } + + final MemoryRange cursor = appendableMemory.cursor(); + final WritableMemory memory = cursor.memory(); + final long position = cursor.start(); + + if (hasNulls) { + memory.putByte(position, (byte) 0); + memory.putLong(position + 1, value); + } else { + memory.putLong(position, value); + } + + appendableMemory.advanceCursor(sz); + return true; + } + + public boolean addNull() + { + if (!hasNulls) { + throw new ISE("Was told that null doesn't exist, cannot add null"); + } + + if (!(appendableMemory.reserveAdditional(sz))) { + return false; + } + + final MemoryRange cursor = appendableMemory.cursor(); + final WritableMemory memory = cursor.memory(); + final long position = cursor.start(); + + memory.putByte(position, (byte) 1); + memory.putLong(position + 1, 0); + + appendableMemory.advanceCursor(sz); + return true; + } + + public void undo() + { + appendableMemory.rewindCursor(sz); + } + + public long size() + { + return DATA_OFFSET + appendableMemory.size(); + } + + public long writeTo(final WritableMemory memory, final long startPosition) + { + long currentPosition = startPosition; + + memory.putByte(currentPosition, FrameColumnWriters.TYPE_LONG); + memory.putByte(currentPosition + 1, hasNulls ? (byte) 1 : (byte) 0); + currentPosition += 2; + + currentPosition += appendableMemory.writeTo(memory, currentPosition); + return currentPosition - startPosition; + } + + @Override + public void close() + { + appendableMemory.close(); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/InlineDataSource.java b/processing/src/main/java/org/apache/druid/query/InlineDataSource.java index 040d6885782a..b3c602403913 100644 --- a/processing/src/main/java/org/apache/druid/query/InlineDataSource.java +++ b/processing/src/main/java/org/apache/druid/query/InlineDataSource.java @@ -32,6 +32,7 @@ import org.apache.druid.segment.column.RowSignature; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -70,7 +71,7 @@ private InlineDataSource( private static InlineDataSource fromJson( @JsonProperty("columnNames") List columnNames, @JsonProperty("columnTypes") List columnTypes, - @JsonProperty("rows") List rows + @JsonProperty("rows") ArrayList rows ) { Preconditions.checkNotNull(columnNames, "'columnNames' must be nonnull"); @@ -200,6 +201,11 @@ public Iterable getRows() return rows; } + public boolean rowsAreArrayList() + { + return rows instanceof ArrayList; + } + @Override public List getChildren() { diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index 9d38dbe37169..90b43469dce5 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -31,6 +31,7 @@ import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.operator.WindowOperatorQuery; import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.search.SearchQuery; import org.apache.druid.query.select.SelectQuery; @@ -45,7 +46,6 @@ import org.joda.time.Interval; import javax.annotation.Nullable; - import java.util.List; import java.util.Map; import java.util.Set; @@ -54,27 +54,29 @@ @ExtensionPoint @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "queryType") @JsonSubTypes(value = { - @JsonSubTypes.Type(name = Query.TIMESERIES, value = TimeseriesQuery.class), - @JsonSubTypes.Type(name = Query.SEARCH, value = SearchQuery.class), - @JsonSubTypes.Type(name = Query.TIME_BOUNDARY, value = TimeBoundaryQuery.class), + @JsonSubTypes.Type(name = Query.DATASOURCE_METADATA, value = DataSourceMetadataQuery.class), @JsonSubTypes.Type(name = Query.GROUP_BY, value = GroupByQuery.class), @JsonSubTypes.Type(name = Query.SCAN, value = ScanQuery.class), + @JsonSubTypes.Type(name = Query.SEARCH, value = SearchQuery.class), @JsonSubTypes.Type(name = Query.SEGMENT_METADATA, value = SegmentMetadataQuery.class), @JsonSubTypes.Type(name = Query.SELECT, value = SelectQuery.class), + @JsonSubTypes.Type(name = Query.TIME_BOUNDARY, value = TimeBoundaryQuery.class), + @JsonSubTypes.Type(name = Query.TIMESERIES, value = TimeseriesQuery.class), @JsonSubTypes.Type(name = Query.TOPN, value = TopNQuery.class), - @JsonSubTypes.Type(name = Query.DATASOURCE_METADATA, value = DataSourceMetadataQuery.class) + @JsonSubTypes.Type(name = Query.WINDOW_OPERATOR, value = WindowOperatorQuery.class), }) public interface Query { - String TIMESERIES = "timeseries"; - String SEARCH = "search"; - String TIME_BOUNDARY = "timeBoundary"; + String DATASOURCE_METADATA = "dataSourceMetadata"; String GROUP_BY = "groupBy"; String SCAN = "scan"; + String SEARCH = "search"; String SEGMENT_METADATA = "segmentMetadata"; String SELECT = "select"; + String TIME_BOUNDARY = "timeBoundary"; + String TIMESERIES = "timeseries"; String TOPN = "topN"; - String DATASOURCE_METADATA = "dataSourceMetadata"; + String WINDOW_OPERATOR = "windowOperator"; DataSource getDataSource(); diff --git a/processing/src/main/java/org/apache/druid/query/operator/LimitedRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/operator/LimitedRowsAndColumns.java new file mode 100644 index 000000000000..d0521dbad0ba --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/LimitedRowsAndColumns.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.operator.window.value.ShiftedColumnAccessorBase; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; + +import javax.annotation.Nullable; +import java.util.Collection; + +public class LimitedRowsAndColumns implements RowsAndColumns +{ + private final RowsAndColumns rac; + private final int start; + private final int end; + + public LimitedRowsAndColumns(RowsAndColumns rac, int start, int end) + { + final int numRows = rac.numRows(); + if (numRows < end) { + throw new ISE("end[%d] is out of bounds, cannot be greater than numRows[%d]", end, numRows); + } + + this.rac = rac; + this.start = start; + this.end = end; + } + + @Override + public Collection getColumnNames() + { + return rac.getColumnNames(); + } + + @Override + public int numRows() + { + return end - start; + } + + @Override + public Column findColumn(String name) + { + final Column column = rac.findColumn(name); + if (column == null) { + return null; + } + + return new Column() + { + @Override + public ColumnAccessor toAccessor() + { + final ColumnAccessor columnAccessor = column.toAccessor(); + return new ShiftedColumnAccessorBase(columnAccessor) + { + @Override + public int numRows() + { + return end - start; + } + + @Override + protected int getActualCell(int cell) + { + int retVal = start + cell; + if (retVal >= end) { + throw new ISE("Index out of bounds[%d] >= [%d], start[%s]", retVal, end, start); + } + return retVal; + } + + @Override + protected boolean outsideBounds(int cell) + { + return false; + } + }; + } + + @Override + public T as(Class clazz) + { + return null; + } + }; + } + + @Nullable + @Override + public T as(Class clazz) + { + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/NaivePartitioningOperator.java b/processing/src/main/java/org/apache/druid/query/operator/NaivePartitioningOperator.java new file mode 100644 index 000000000000..8063a78b5984 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/NaivePartitioningOperator.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.rowsandcols.DefaultSortedGroupPartitioner; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.SortedGroupPartitioner; + +import java.util.Iterator; +import java.util.List; + +/** + * This naive partitioning operator assumes that it's child operator always gives it RowsAndColumns objects that are + * a superset of the partitions that it needs to provide. It will never attempt to make a partition larger than a + * single RowsAndColumns object that it is given from its child Operator. A different operator should be used + * if that is an important bit of functionality to have. + *

+ * Additionally, this assumes that data has been pre-sorted according to the partitioning columns. If it is + * given data that has not been pre-sorted, an exception is expected to be thrown. + */ +public class NaivePartitioningOperator implements Operator +{ + private final List partitionColumns; + private final Operator child; + + private Iterator partitionsIter; + + public NaivePartitioningOperator( + List partitionColumns, + Operator child + ) + { + this.partitionColumns = partitionColumns; + this.child = child; + } + + @Override + public void open() + { + child.open(); + } + + @Override + public RowsAndColumns next() + { + if (partitionsIter != null && partitionsIter.hasNext()) { + return partitionsIter.next(); + } + + if (child.hasNext()) { + final RowsAndColumns rac = child.next(); + + SortedGroupPartitioner groupPartitioner = rac.as(SortedGroupPartitioner.class); + if (groupPartitioner == null) { + groupPartitioner = new DefaultSortedGroupPartitioner(rac); + } + + partitionsIter = groupPartitioner.partitionOnBoundaries(partitionColumns).iterator(); + return partitionsIter.next(); + } + + throw new ISE("Asked for next when already complete"); + } + + @Override + public boolean hasNext() + { + if (partitionsIter != null && partitionsIter.hasNext()) { + return true; + } + + return child.hasNext(); + } + + @Override + public void close(boolean cascade) + { + if (cascade) { + child.close(cascade); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/NaivePartitioningOperatorFactory.java b/processing/src/main/java/org/apache/druid/query/operator/NaivePartitioningOperatorFactory.java new file mode 100644 index 000000000000..9bd937ab8443 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/NaivePartitioningOperatorFactory.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; + +public class NaivePartitioningOperatorFactory implements OperatorFactory +{ + private final List partitionColumns; + + @JsonCreator + public NaivePartitioningOperatorFactory( + @JsonProperty("partitionColumns") List partitionColumns + ) + { + this.partitionColumns = partitionColumns == null ? new ArrayList<>() : partitionColumns; + } + + @JsonProperty("partitionColumns") + public List getPartitionColumns() + { + return partitionColumns; + } + + @Override + public Operator wrap(Operator op) + { + return new NaivePartitioningOperator(partitionColumns, op); + } + + @Override + public boolean validateEquivalent(OperatorFactory other) + { + if (other instanceof NaivePartitioningOperatorFactory) { + return partitionColumns.equals(((NaivePartitioningOperatorFactory) other).getPartitionColumns()); + } + return false; + } + + @Override + public String toString() + { + return "NaivePartitioningOperatorFactory{" + + "partitionColumns=" + partitionColumns + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/Operator.java b/processing/src/main/java/org/apache/druid/query/operator/Operator.java new file mode 100644 index 000000000000..ad64cd1a5235 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/Operator.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.query.rowsandcols.RowsAndColumns; + +/** + * An Operator interface that intends to align closely with the Operators that other databases would also tend + * to be implemented using. + *

+ * The lifecycle of an operator is that, after creation, it should be opened, and then iterated using hasNext() and + * next(). Finally, when the Operator is no longer useful, it should be closed. + *

+ * Operator's methods mimic the methods of an {@code Iterator}, but it does not implement {@code Iterator} + * intentionally. An operator should never be wrapped in an {@code Iterator}. Any code that does that should be + * considered a bug and fixed. This is for two reasons: + *

+ * 1. An Operator should never be passed around as an {@code Iterator}. An Operator must be closed, if an operator + * gets returned as an {@code Iterator}, the code that sees the {@code Iterator} loses the knowledge that it's + * dealing with an Operator and might not close it. Even something like a {@code CloseableIterator} is an + * anti-pattern as it's possible to use it in a functional manner with code that loses track of the fact that it + * must be closed. + * 2. To avoid "fluent" style composition of functions on Operators. It is important that there never be a set of + * functional primitives for things like map/filter/reduce to "simplify" the implementation of Operators. This is + * because such fluency produces really hard to decipher stack traces as the stacktrace ends up being just a bunch + * of calls from the scaffolding (map/filter/reduce) and not from the actual Operator itself. By not implementing + * {@code Iterator} we are actively increasing the burden of trying to add such functional operations to the point + * that hopefully, though code review, we can ensure that we never develop them. It is infinitely better to preserve + * the stacktrace and "duplicate" the map/filter/reduce scaffolding code. + */ +public interface Operator +{ + /** + * Called to initiate the lifecycle of the Operator. If an operator needs to checkout resources or anything to do + * its work, this is probably the place to do it. + * + * Work should *never* be done in this method, this method only exists to acquire resources that are known to be + * needed before doing any work. As a litmus test, if there is ever a call to `op.next()` inside of this method, + * then something has been done wrong as that call to `.next()` is actually doing work. Such code should be moved + * into being lazily evaluated as part of a call to `.next()`. + */ + void open(); + + /** + * Returns the next RowsAndColumns object that the Operator can produce. Behavior is undefined if + * {@link #hasNext} returns false. + * + * @return the next RowsAndColumns object that the operator can produce + */ + RowsAndColumns next(); + + /** + * Used to identify if it is safe to call {@link #next} + * + * @return true if it is safe to call {@link #next} + */ + boolean hasNext(); + + /** + * Closes this Operator. The cascade flag can be used to identify that the intent is to close this operator + * and only this operator without actually closing child operators. Other databases us this sort of functionality + * with a planner that is watching over all of the objects and force-closes even if they were closed during normal + * operations. In Druid, in the data pipeline where this was introduced, we are guaranteed to always have close + * called regardless of errors or exceptions during processing, as such, at time of introduction, there is no + * call that passes false for cascade. + *

+ * That said, given that this is a common thing for these interfaces for other databases, we want to preserve the + * optionality of being able to leverage what they do. As such, we define the method this way with the belief + * that it might be used in the future. Semantically, this means that all implementations of Operators must + * expect to be closed multiple times. I.e. after being closed, it is an error for open, next or hasNext to be + * called, but close can be called any number of times. + * + * @param cascade whether to call close on child operators. + */ + void close(boolean cascade); +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/OperatorFactory.java b/processing/src/main/java/org/apache/druid/query/operator/OperatorFactory.java new file mode 100644 index 000000000000..26235c35d8d8 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/OperatorFactory.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * A factory for Operators. This class exists to encapsulate the user-definition of an Operator. I.e. which operator, + * what fields it should operate on, etc. etc. These Factory objects are then used to combine Operators together + * and run against concrete data. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "naivePartition", value = NaivePartitioningOperatorFactory.class), + @JsonSubTypes.Type(name = "window", value = WindowOperatorFactory.class), +}) +public interface OperatorFactory +{ + /** + * Builds an operator according to the definition of the OperatorFactory and wraps it around the operator passed + * in to this function. + * + * @param op the Operator to wrap + * @return the wrapped Operator + */ + Operator wrap(Operator op); + + /** + * Validates the equivalence of Operators. This is similar to @{code .equals} but is its own method + * so that it can ignore certain fields that would be important for a true equality check. Namely, two Operators + * defined the same way but with different output names can be considered equivalent even though they are not equal. + *

+ * This primarily exists to simplify tests, where this equivalence can be used to validate that the Operators + * created by the SQL planner are actually equivalent to what we expect without needing to be overly dependent on + * how the planner names output columns + * + * @param other the processor to test equivalence of + * @return boolean identifying if these processors should be considered equivalent to each other. + */ + boolean validateEquivalent(OperatorFactory other); +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/OperatorSequence.java b/processing/src/main/java/org/apache/druid/query/operator/OperatorSequence.java new file mode 100644 index 000000000000..45a3bbd23899 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/OperatorSequence.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.java.util.common.guava.Accumulator; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Yielder; +import org.apache.druid.java.util.common.guava.Yielders; +import org.apache.druid.java.util.common.guava.YieldingAccumulator; +import org.apache.druid.query.rowsandcols.RowsAndColumns; + +import java.util.function.Supplier; + +public class OperatorSequence implements Sequence +{ + private final Supplier opSupplier; + + public OperatorSequence( + Supplier opSupplier + ) + { + this.opSupplier = opSupplier; + } + + @Override + public OutType accumulate( + OutType initValue, + Accumulator accumulator + ) + { + Operator op = null; + try { + op = opSupplier.get(); + op.open(); + while (op.hasNext()) { + initValue = accumulator.accumulate(initValue, op.next()); + } + return initValue; + } + finally { + if (op != null) { + op.close(true); + } + } + } + + @Override + public Yielder toYielder( + OutType initValue, + YieldingAccumulator accumulator + ) + { + final Operator op = opSupplier.get(); + try { + op.open(); + + while (!accumulator.yielded() && op.hasNext()) { + initValue = accumulator.accumulate(initValue, op.next()); + } + if (accumulator.yielded()) { + OutType finalInitValue = initValue; + return new Yielder() + { + private OutType retVal = finalInitValue; + private boolean done = false; + + @Override + public OutType get() + { + return retVal; + } + + @Override + public Yielder next(OutType initValue) + { + accumulator.reset(); + retVal = initValue; + while (!accumulator.yielded() && op.hasNext()) { + retVal = accumulator.accumulate(retVal, op.next()); + } + if (!accumulator.yielded()) { + done = true; + } + return this; + } + + @Override + public boolean isDone() + { + return done; + } + + @Override + public void close() + { + op.close(true); + } + }; + } else { + return Yielders.done(initValue, () -> op.close(true)); + } + } + catch (RuntimeException e) { + op.close(true); + throw e; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/SegmentToRowsAndColumnsOperator.java b/processing/src/main/java/org/apache/druid/query/operator/SegmentToRowsAndColumnsOperator.java new file mode 100644 index 000000000000..dc912bfdd6f6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/SegmentToRowsAndColumnsOperator.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.segment.Segment; + +public class SegmentToRowsAndColumnsOperator implements Operator +{ + private final Segment segment; + private boolean hasNext = true; + + public SegmentToRowsAndColumnsOperator( + Segment segment + ) + { + this.segment = segment; + } + + @Override + public void open() + { + + } + + @Override + public RowsAndColumns next() + { + hasNext = false; + + RowsAndColumns rac = segment.as(RowsAndColumns.class); + if (rac != null) { + return rac; + } + + throw new ISE("Cannot work with segment of type[%s]", segment.getClass()); + } + + @Override + public boolean hasNext() + { + return hasNext; + } + + @Override + public void close(boolean cascade) + { + + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/SequenceOperator.java b/processing/src/main/java/org/apache/druid/query/operator/SequenceOperator.java new file mode 100644 index 000000000000..9dc54f9576c8 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/SequenceOperator.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Yielder; +import org.apache.druid.java.util.common.guava.Yielders; +import org.apache.druid.query.rowsandcols.RowsAndColumns; + +import java.io.IOException; +import java.util.NoSuchElementException; + +public class SequenceOperator implements Operator +{ + private final Sequence child; + private Yielder yielder; + private boolean closed = false; + + public SequenceOperator( + Sequence child + ) + { + this.child = child; + } + + @Override + public void open() + { + if (closed) { + throw new ISE("Operator closed, cannot be re-opened"); + } + yielder = Yielders.each(child); + } + + @Override + public RowsAndColumns next() + { + if (closed) { + throw new NoSuchElementException(); + } + final RowsAndColumns retVal = yielder.get(); + yielder = yielder.next(null); + return retVal; + } + + @Override + public boolean hasNext() + { + return !closed && !yielder.isDone(); + } + + @Override + public void close(boolean cascade) + { + if (closed) { + return; + } + try { + yielder.close(); + } + catch (IOException e) { + throw new RE(e, "Exception when closing yielder from Sequence"); + } + finally { + closed = true; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorFactory.java b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorFactory.java new file mode 100644 index 000000000000..bc4cd5206c26 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorFactory.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.query.operator.window.Processor; + +public class WindowOperatorFactory implements OperatorFactory +{ + private Processor processor; + + @JsonCreator + public WindowOperatorFactory( + @JsonProperty("processor") Processor processor + ) + { + Preconditions.checkNotNull(processor, "processor cannot be null"); + this.processor = processor; + } + + @JsonProperty("processor") + public Processor getProcessor() + { + return processor; + } + + @Override + public Operator wrap(Operator op) + { + return new WindowProcessorOperator(processor, op); + } + + @Override + public boolean validateEquivalent(OperatorFactory other) + { + if (other instanceof WindowOperatorFactory) { + return processor.validateEquivalent(((WindowOperatorFactory) other).getProcessor()); + } + return false; + } + + @Override + public String toString() + { + return "WindowOperatorFactory{" + + "processor=" + processor + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java new file mode 100644 index 000000000000..99faf8068f38 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.UOE; +import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryDataSource; +import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.spec.LegacySegmentSpec; +import org.apache.druid.query.spec.QuerySegmentSpec; +import org.apache.druid.segment.column.RowSignature; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A query that can compute window functions on top of a completely in-memory inline datasource or query results. + *

+ * It relies on a set of Operators to work on the data that it is given. As such, it doesn't actually encapsulate + * any window-specific logic in-and-of-itself, but rather delegates everything to the operators. This is because + * this is also intended as the initial addition of more explicit Operators to the Druid code base. + *

+ * The assumptions on the incoming data are defined by the operators. At initial time of writing, there is a baked + * in assumption that data has been sorted "correctly" before this runs. + */ +public class WindowOperatorQuery extends BaseQuery +{ + private final RowSignature rowSignature; + private final List operators; + + @JsonCreator + public WindowOperatorQuery( + @JsonProperty("dataSource") DataSource dataSource, + @JsonProperty("context") Map context, + @JsonProperty("outputSignature") RowSignature rowSignature, + @JsonProperty("operatorDefinition") List operators + ) + { + super(dataSource, new LegacySegmentSpec(Intervals.ETERNITY), false, context); + this.rowSignature = rowSignature; + this.operators = operators; + if (!(dataSource instanceof QueryDataSource || dataSource instanceof InlineDataSource)) { + throw new IAE("WindowOperatorQuery must run on top of a query or inline data source, got [%s]", dataSource); + } + } + + @JsonProperty("operatorDefinition") + public List getOperators() + { + return operators; + } + + @JsonProperty("outputSignature") + public RowSignature getRowSignature() + { + return rowSignature; + } + + @Override + public boolean hasFilters() + { + return false; + } + + @Override + public DimFilter getFilter() + { + return null; + } + + @Override + public String getType() + { + return Query.WINDOW_OPERATOR; + } + + @Override + public Query withOverriddenContext(Map contextOverride) + { + return new WindowOperatorQuery( + getDataSource(), + computeOverriddenContext(getContext(), contextOverride), + rowSignature, + operators + ); + } + + @Override + public Query withQuerySegmentSpec(QuerySegmentSpec spec) + { + throw new UOE("Cannot override querySegmentSpec on window operator query. [%s]", spec); + } + + @Override + public Query withDataSource(DataSource dataSource) + { + return new WindowOperatorQuery( + dataSource, + getContext(), + rowSignature, + operators + ); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + WindowOperatorQuery that = (WindowOperatorQuery) o; + return Objects.equals(rowSignature, that.rowSignature) && Objects.equals( + operators, + that.operators + ); + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), rowSignature, operators); + } + + @Override + public String toString() + { + return "WindowOperatorQuery{" + + "dataSource='" + getDataSource() + '\'' + + ", querySegmentSpec=" + getQuerySegmentSpec() + + ", context=" + getContext() + + ", rowSignature=" + rowSignature + + ", operators=" + operators + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQueryQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQueryQueryRunnerFactory.java new file mode 100644 index 000000000000..78f64360761e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQueryQueryRunnerFactory.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.google.common.collect.Iterables; +import org.apache.druid.query.QueryProcessingPool; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerFactory; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.segment.Segment; + +public class WindowOperatorQueryQueryRunnerFactory implements QueryRunnerFactory +{ + public static final WindowOperatorQueryQueryToolChest TOOLCHEST = new WindowOperatorQueryQueryToolChest(); + + @Override + public QueryRunner createRunner(Segment segment) + { + return (queryPlus, responseContext) -> + new OperatorSequence(() -> new SegmentToRowsAndColumnsOperator(segment)); + } + + @Override + public QueryRunner mergeRunners( + QueryProcessingPool queryProcessingPool, + Iterable> queryRunners + ) + { + return Iterables.getOnlyElement(queryRunners); + } + + @Override + public QueryToolChest getToolchest() + { + return TOOLCHEST; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQueryQueryToolChest.java new file mode 100644 index 000000000000..d02596900df0 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQueryQueryToolChest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.DefaultQueryMetrics; +import org.apache.druid.query.QueryMetrics; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.aggregation.MetricManipulationFn; +import org.apache.druid.query.context.ResponseContext; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.query.rowsandcols.column.NullColumnAccessor; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class WindowOperatorQueryQueryToolChest extends QueryToolChest +{ + + @Override + @SuppressWarnings("unchecked") + public QueryRunner mergeResults(QueryRunner runner) + { + return new RowsAndColumnsUnravelingQueryRunner( + (queryPlus, responseContext) -> { + final WindowOperatorQuery query = (WindowOperatorQuery) queryPlus.getQuery(); + final List opFactories = query.getOperators(); + + Supplier opSupplier = () -> { + Operator retVal = new SequenceOperator(runner.run(queryPlus, responseContext)); + for (OperatorFactory operatorFactory : opFactories) { + retVal = operatorFactory.wrap(retVal); + } + return retVal; + }; + + return new OperatorSequence(opSupplier); + } + ); + } + + @Override + public QueryMetrics makeMetrics(WindowOperatorQuery query) + { + return new DefaultQueryMetrics<>(); + } + + @Override + public Function makePreComputeManipulatorFn( + WindowOperatorQuery query, + MetricManipulationFn fn + ) + { + return Functions.identity(); + } + + @Override + public TypeReference getResultTypeReference() + { + return new TypeReference() + { + }; + } + + @Override + public RowSignature resultArraySignature(WindowOperatorQuery query) + { + return query.getRowSignature(); + } + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public Sequence resultsAsArrays( + WindowOperatorQuery query, + Sequence resultSequence + ) + { + // Dark magic; see RowsAndColumnsUnravelingQueryRunner. + return (Sequence) resultSequence; + } + + /** + * This class exists to unravel the RowsAndColumns that are used in this query and make it the return Sequence + * actually be a Sequence of rows. This is relatively broken in a number of regards, the most obvious of which + * is that it is going to run counter to the stated class on the Generic of the QueryToolChest. That is, the + * code makes it look like you are getting a Sequence of RowsAndColumns, but, by using this, the query will + * actually ultimately produce a Sequence of Object[]. This works because of type Erasure in Java (it's all Object + * at the end of the day). + *

+ * While it might seem like this will break all sorts of things, the Generic type is actually there more as a type + * "hint" to make the writing of the ToolChest and Factory and stuff a bit more simple. Any caller of this cannot + * truly depend on the type anyway other than to just throw it across the wire, so this should just magically work + * even though it looks like it shouldn't even compile. + *

+ * Not our proudest moment, but we use the tools available to us. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + private static class RowsAndColumnsUnravelingQueryRunner implements QueryRunner + { + + private final QueryRunner baseQueryRunner; + + private RowsAndColumnsUnravelingQueryRunner( + QueryRunner baseQueryRunner + ) + { + this.baseQueryRunner = baseQueryRunner; + } + + @Override + public Sequence run( + QueryPlus queryPlus, + ResponseContext responseContext + ) + { + // We only want to do this operation once at the very, very top of the execution tree. So we check and set + // a context parameter so that if this merge code runs anywhere else, it will skip this part. + final WindowOperatorQuery query = (WindowOperatorQuery) queryPlus.getQuery(); + if (query.context().getBoolean("unravel", true)) { + final Sequence baseSequence = baseQueryRunner.run( + queryPlus.withQuery(query.withOverriddenContext(ImmutableMap.of("unravel", false))), + responseContext + ); + + final RowSignature rowSignature = query.getRowSignature(); + return baseSequence.flatMap( + rac -> { + List results = new ArrayList<>(rac.numRows()); + + ColumnAccessor[] accessors = new ColumnAccessor[rowSignature.size()]; + int index = 0; + for (String columnName : rowSignature.getColumnNames()) { + final Column column = rac.findColumn(columnName); + if (column == null) { + final ColumnType columnType = rowSignature + .getColumnType(columnName) + .orElse(ColumnType.UNKNOWN_COMPLEX); + + accessors[index] = new NullColumnAccessor(columnType, rac.numRows()); + } else { + accessors[index] = column.toAccessor(); + } + ++index; + } + + for (int i = 0; i < rac.numRows(); ++i) { + Object[] objArr = new Object[accessors.length]; + for (int j = 0; j < accessors.length; j++) { + objArr[j] = accessors[j].getObject(i); + } + results.add(objArr); + } + + return Sequences.simple(results); + } + ); + } + + return baseQueryRunner.run(queryPlus, responseContext); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowProcessorOperator.java b/processing/src/main/java/org/apache/druid/query/operator/WindowProcessorOperator.java new file mode 100644 index 000000000000..d95f598d3993 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/WindowProcessorOperator.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.rowsandcols.RowsAndColumns; + +/** + * An Operator that applies a {@link Processor}, see javadoc on that interface for an explanation. + */ +public class WindowProcessorOperator implements Operator +{ + private final Processor windowProcessor; + private final Operator child; + + public WindowProcessorOperator( + Processor windowProcessor, + Operator child + ) + { + this.windowProcessor = windowProcessor; + this.child = child; + } + + @Override + public void open() + { + child.open(); + } + + @Override + public RowsAndColumns next() + { + return windowProcessor.process(child.next()); + } + + @Override + public boolean hasNext() + { + return child.hasNext(); + } + + @Override + public void close(boolean cascade) + { + if (cascade) { + child.close(cascade); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/ComposingProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/ComposingProcessor.java new file mode 100644 index 000000000000..a4fa74967f61 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/ComposingProcessor.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.rowsandcols.RowsAndColumns; + +import java.util.Arrays; + +public class ComposingProcessor implements Processor +{ + private final Processor[] processors; + + @JsonCreator + public ComposingProcessor( + @JsonProperty("processors") Processor... processors + ) + { + this.processors = processors; + } + + @JsonProperty("processors") + public Processor[] getProcessors() + { + return processors; + } + + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + RowsAndColumns retVal = incomingPartition; + for (int i = processors.length - 1; i >= 0; --i) { + retVal = processors[i].process(retVal); + } + return retVal; + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + if (otherProcessor instanceof ComposingProcessor) { + ComposingProcessor other = (ComposingProcessor) otherProcessor; + for (int i = 0; i < processors.length; ++i) { + if (!processors[i].validateEquivalent(other.processors[i])) { + return false; + } + } + return true; + } + return false; + } + + @Override + public String toString() + { + return "ComposingProcessor{" + + "processors=" + Arrays.toString(processors) + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/Processor.java b/processing/src/main/java/org/apache/druid/query/operator/window/Processor.java new file mode 100644 index 000000000000..94e8c74a6b50 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/Processor.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.query.operator.window.ranking.WindowCumeDistProcessor; +import org.apache.druid.query.operator.window.ranking.WindowDenseRankProcessor; +import org.apache.druid.query.operator.window.ranking.WindowPercentileProcessor; +import org.apache.druid.query.operator.window.ranking.WindowRankProcessor; +import org.apache.druid.query.operator.window.ranking.WindowRowNumberProcessor; +import org.apache.druid.query.operator.window.value.WindowFirstProcessor; +import org.apache.druid.query.operator.window.value.WindowLastProcessor; +import org.apache.druid.query.operator.window.value.WindowOffsetProcessor; +import org.apache.druid.query.rowsandcols.RowsAndColumns; + +/** + * A Processor is a bit of logic that processes a single RowsAndColumns object to produce a new RowsAndColumns + * object. Generally speaking, it is used to add or alter columns in a batch-oriented fashion. + *

+ * This interface was created to support windowing functions, where the windowing function can be implemented + * assuming that each RowsAndColumns object represents one partition. Thus, the window function implementation + * can only need to worry about how to process a single partition at a time and something external to the window + * function worries about providing data with the correct partitioning. + *

+ * Over time, it's possible that this interface is used for other purposes as well, but the fundamental idea of + * usages of the interface should always be doing a one-to-one transformation of RowsAndColumns objects. That is, + * it's a RowsAndColumns in and a RowsAndColumns out. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "composing", value = ComposingProcessor.class), + @JsonSubTypes.Type(name = "cumeDist", value = WindowCumeDistProcessor.class), + @JsonSubTypes.Type(name = "denseRank", value = WindowDenseRankProcessor.class), + @JsonSubTypes.Type(name = "percentile", value = WindowPercentileProcessor.class), + @JsonSubTypes.Type(name = "rank", value = WindowRankProcessor.class), + @JsonSubTypes.Type(name = "rowNumber", value = WindowRowNumberProcessor.class), + @JsonSubTypes.Type(name = "first", value = WindowFirstProcessor.class), + @JsonSubTypes.Type(name = "last", value = WindowLastProcessor.class), + @JsonSubTypes.Type(name = "offset", value = WindowOffsetProcessor.class), + @JsonSubTypes.Type(name = "aggregate", value = WindowAggregateProcessor.class), +}) +public interface Processor +{ + /** + * Applies the logic of the processor to a RowsAndColumns object + * + * @param incomingPartition the incoming RowsAndColumns object + * @return the transformed RowsAndColumns object + */ + RowsAndColumns process(RowsAndColumns incomingPartition); + + /** + * Validates the equivalence of the Processors. This is similar to @{code .equals} but is its own method + * so that it can ignore certain fields that would be important for a true equality check. Namely, two Processors + * defined the same way but with different output names can be considered equivalent even though they are not equal. + *

+ * This primarily exists to simplify tests, where this equivalence can be used to validate that the Processors + * created by the SQL planner are actually equivalent to what we expect without needing to be overly dependent on + * how the planner names the output columns + * + * @param otherProcessor the processor to test equivalence of + * @return boolean identifying if these processors should be considered equivalent to each other. + */ + boolean validateEquivalent(Processor otherProcessor); +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/WindowAggregateProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/WindowAggregateProcessor.java new file mode 100644 index 000000000000..630d1145f770 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/WindowAggregateProcessor.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.rowsandcols.AppendableRowsAndColumns; +import org.apache.druid.query.rowsandcols.DefaultOnHeapAggregatable; +import org.apache.druid.query.rowsandcols.OnHeapAggregatable; +import org.apache.druid.query.rowsandcols.OnHeapCumulativeAggregatable; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.ConstantObjectColumn; +import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class WindowAggregateProcessor implements Processor +{ + @Nullable + private static List emptyToNull(List list) + { + if (list == null || list.isEmpty()) { + return null; + } else { + return list; + } + } + + private final List aggregations; + private final List cumulativeAggregations; + + @JsonCreator + public WindowAggregateProcessor( + @JsonProperty("aggregations") List aggregations, + @JsonProperty("cumulativeAggregations") List cumulativeAggregations + ) + { + this.aggregations = emptyToNull(aggregations); + this.cumulativeAggregations = emptyToNull(cumulativeAggregations); + } + + @JsonProperty("aggregations") + public List getAggregations() + { + return aggregations; + } + + @JsonProperty("cumulativeAggregations") + public List getCumulativeAggregations() + { + return cumulativeAggregations; + } + + @Override + public RowsAndColumns process(RowsAndColumns inputPartition) + { + AppendableRowsAndColumns retVal = RowsAndColumns.expectAppendable(inputPartition); + + if (aggregations != null) { + OnHeapAggregatable aggregatable = inputPartition.as(OnHeapAggregatable.class); + if (aggregatable == null) { + aggregatable = new DefaultOnHeapAggregatable(inputPartition); + } + final ArrayList aggregatedVals = aggregatable.aggregateAll(aggregations); + + for (int i = 0; i < aggregations.size(); ++i) { + final AggregatorFactory agg = aggregations.get(i); + retVal.addColumn( + agg.getName(), + new ConstantObjectColumn(aggregatedVals.get(i), inputPartition.numRows(), agg.getResultType()) + ); + } + } + + if (cumulativeAggregations != null) { + OnHeapCumulativeAggregatable cummulativeAgg = inputPartition.as(OnHeapCumulativeAggregatable.class); + if (cummulativeAgg == null) { + cummulativeAgg = new DefaultOnHeapAggregatable(inputPartition); + } + final ArrayList cumulativeVals = cummulativeAgg.aggregateCumulative(cumulativeAggregations); + + for (int i = 0; i < cumulativeAggregations.size(); ++i) { + final AggregatorFactory agg = cumulativeAggregations.get(i); + retVal.addColumn(agg.getName(), new ObjectArrayColumn(cumulativeVals.get(i), agg.getResultType())); + } + } + + return retVal; + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + if (otherProcessor instanceof WindowAggregateProcessor) { + WindowAggregateProcessor other = (WindowAggregateProcessor) otherProcessor; + return Objects.equals(aggregations, other.aggregations) + && Objects.equals(cumulativeAggregations, other.cumulativeAggregations); + } + return false; + } + + @Override + public String toString() + { + return "WindowAggregateProcessor{" + + "aggregations=" + aggregations + + ", cumulativeAggregations=" + cumulativeAggregations + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowCumeDistProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowCumeDistProcessor.java new file mode 100644 index 000000000000..c798081339e7 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowCumeDistProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; + +import java.util.Arrays; +import java.util.List; + +/** + * This Processor assumes that data has already been sorted for it. It does not re-sort the data and if it is given + * data that is not in the correct sorted order, its operation is undefined. + */ +public class WindowCumeDistProcessor extends WindowRankingProcessorBase +{ + @JsonCreator + public WindowCumeDistProcessor( + @JsonProperty("group") List groupingCols, + @JsonProperty("outputColumn") String outputColumn + ) + { + super(groupingCols, outputColumn); + } + + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + return processInternal(incomingPartition, groupings -> { + final double[] ranks = new double[incomingPartition.numRows()]; + for (int i = 1; i < groupings.length; ++i) { + final int start = groupings[i - 1]; + final int end = groupings[i]; + double relativeRank = end / (double) ranks.length; + Arrays.fill(ranks, start, end, relativeRank); + } + + return new DoubleArrayColumn(ranks); + }); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowDenseRankProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowDenseRankProcessor.java new file mode 100644 index 000000000000..24b05a6f3460 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowDenseRankProcessor.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; + +import java.util.Arrays; +import java.util.List; + +/** + * This Processor assumes that data has already been sorted for it. It does not re-sort the data and if it is given + * data that is not in the correct sorted order, its operation is undefined. + */ +public class WindowDenseRankProcessor extends WindowRankingProcessorBase +{ + @JsonCreator + public WindowDenseRankProcessor( + @JsonProperty("group") List groupingCols, + @JsonProperty("outputColumn") String outputColumn + ) + { + super(groupingCols, outputColumn); + } + + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + return processInternal(incomingPartition, groupings -> { + final int[] ranks = new int[incomingPartition.numRows()]; + for (int i = 1; i < groupings.length; ++i) { + final int start = groupings[i - 1]; + final int end = groupings[i]; + Arrays.fill(ranks, start, end, i); + } + + return new IntArrayColumn(ranks); + }); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowPercentileProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowPercentileProcessor.java new file mode 100644 index 000000000000..e8c9aec2ab4c --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowPercentileProcessor.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.rowsandcols.AppendableRowsAndColumns; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; + +import java.util.Arrays; + +public class WindowPercentileProcessor implements Processor +{ + private final int numBuckets; + private final String outputColumn; + + @JsonCreator + public WindowPercentileProcessor( + @JsonProperty("outputColumn") String outputColumn, + @JsonProperty("numBuckets") int numBuckets + ) + { + Preconditions.checkArgument(numBuckets > 0, "numBuckets[%s] must be greater than zero", numBuckets); + + this.outputColumn = outputColumn; + this.numBuckets = numBuckets; + } + + @JsonProperty("numBuckets") + public int getNumBuckets() + { + return numBuckets; + } + + @JsonProperty("outputColumn") + public String getOutputColumn() + { + return outputColumn; + } + + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + AppendableRowsAndColumns retVal = RowsAndColumns.expectAppendable(incomingPartition); + + int numRows = incomingPartition.numRows(); + int countPerBucket = numRows / numBuckets; + int extraRows = numRows % numBuckets; + + int index = 0; + int[] bucketVals = new int[numRows]; + for (int i = 0; i < numBuckets; ++i) { + int nextIndex = index + countPerBucket; + if (extraRows > 0) { + ++nextIndex; + --extraRows; + } + + // Buckets are 1-indexed, so we fill with i+1 + Arrays.fill(bucketVals, index, nextIndex, i + 1); + + index = nextIndex; + } + + retVal.addColumn(outputColumn, new IntArrayColumn(bucketVals)); + return retVal; + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + if (otherProcessor instanceof WindowPercentileProcessor) { + return numBuckets == ((WindowPercentileProcessor) otherProcessor).numBuckets; + } + return false; + } + + @Override + public String toString() + { + return "WindowPercentileProcessor{" + + "numBuckets=" + numBuckets + + ", outputColumn='" + outputColumn + '\'' + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRankProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRankProcessor.java new file mode 100644 index 000000000000..2d9b21863de8 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRankProcessor.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; + +import java.util.Arrays; +import java.util.List; + +/** + * This Processor assumes that data has already been sorted for it. It does not re-sort the data and if it is given + * data that is not in the correct sorted order, its operation is undefined. + */ +public class WindowRankProcessor extends WindowRankingProcessorBase +{ + private final boolean asPercent; + + @JsonCreator + public WindowRankProcessor( + @JsonProperty("group") List groupingCols, + @JsonProperty("outputColumn") String outputColumn, + @JsonProperty("asPercent") boolean asPercent + ) + { + super(groupingCols, outputColumn); + this.asPercent = asPercent; + } + + @JsonProperty("asPercent") + public boolean isAsPercent() + { + return asPercent; + } + + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + if (asPercent) { + return processInternal(incomingPartition, groupings -> { + final double[] percentages = new double[incomingPartition.numRows()]; + if (percentages.length > 1) { + final double denominator = percentages.length - 1; + + for (int i = 1; i < groupings.length; ++i) { + final int start = groupings[i - 1]; + final int end = groupings[i]; + Arrays.fill(percentages, start, end, start / denominator); + } + } + + return new DoubleArrayColumn(percentages); + }); + } + + return processInternal(incomingPartition, groupings -> { + final int[] ranks = new int[incomingPartition.numRows()]; + + for (int i = 1; i < groupings.length; ++i) { + final int start = groupings[i - 1]; + final int end = groupings[i]; + Arrays.fill(ranks, start, end, start + 1); + } + + return new IntArrayColumn(ranks); + }); + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + if (otherProcessor instanceof WindowRankProcessor) { + WindowRankProcessor other = (WindowRankProcessor) otherProcessor; + return asPercent == other.asPercent && intervalValidation(other); + } + return false; + } + + @Override + public String toString() + { + return "WindowRankProcessor{" + + internalToString() + + ", asPercent=" + asPercent + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRankingProcessorBase.java b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRankingProcessorBase.java new file mode 100644 index 000000000000..105c586aa195 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRankingProcessorBase.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.rowsandcols.AppendableRowsAndColumns; +import org.apache.druid.query.rowsandcols.DefaultSortedGroupPartitioner; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.SortedGroupPartitioner; +import org.apache.druid.query.rowsandcols.column.Column; + +import java.util.List; +import java.util.function.Function; + +/** + * This Processor assumes that data has already been sorted for it. It does not re-sort the data and if it is given + * data that is not in the correct sorted order, its operation is undefined. + */ +public abstract class WindowRankingProcessorBase implements Processor +{ + private final List groupingCols; + private final String outputColumn; + + public WindowRankingProcessorBase( + List groupingCols, + String outputColumn + ) + { + this.groupingCols = groupingCols; + this.outputColumn = outputColumn; + } + + @JsonProperty("group") + public List getGroupingCols() + { + return groupingCols; + } + + @JsonProperty("outputColumn") + public String getOutputColumn() + { + return outputColumn; + } + + public RowsAndColumns processInternal( + RowsAndColumns incomingPartition, + Function fn + ) + { + final AppendableRowsAndColumns retVal = RowsAndColumns.expectAppendable(incomingPartition); + + SortedGroupPartitioner groupPartitioner = incomingPartition.as(SortedGroupPartitioner.class); + if (groupPartitioner == null) { + groupPartitioner = new DefaultSortedGroupPartitioner(incomingPartition); + } + + retVal.addColumn(outputColumn, fn.apply(groupPartitioner.computeBoundaries(groupingCols))); + return retVal; + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + return getClass() == otherProcessor.getClass() + && intervalValidation((WindowRankingProcessorBase) otherProcessor); + } + + protected boolean intervalValidation(WindowRankingProcessorBase other) + { + // Only input needs to be the same for the processors to produce equivalent results + return groupingCols.equals(other.groupingCols); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + internalToString() + '}'; + } + + protected String internalToString() + { + return "groupingCols=" + groupingCols + + ", outputColumn='" + outputColumn + '\''; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRowNumberProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRowNumberProcessor.java new file mode 100644 index 000000000000..6c998afa6099 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/ranking/WindowRowNumberProcessor.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.rowsandcols.AppendableRowsAndColumns; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.query.rowsandcols.column.ColumnAccessorBasedColumn; +import org.apache.druid.segment.column.ColumnType; + +public class WindowRowNumberProcessor implements Processor +{ + private final String outputColumn; + + @JsonCreator + public WindowRowNumberProcessor( + @JsonProperty("outputColumn") String outputColumn + ) + { + this.outputColumn = outputColumn; + } + + @JsonProperty("outputColumn") + public String getOutputColumn() + { + return outputColumn; + } + + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + final AppendableRowsAndColumns retVal = RowsAndColumns.expectAppendable(incomingPartition); + retVal.addColumn( + outputColumn, + new ColumnAccessorBasedColumn( + new ColumnAccessor() + { + @Override + public ColumnType getType() + { + return ColumnType.LONG; + } + + @Override + public int numRows() + { + return incomingPartition.numRows(); + } + + @Override + public boolean isNull(int rowNum) + { + return false; + } + + @Override + public Object getObject(int rowNum) + { + return getInt(rowNum); + } + + @Override + public double getDouble(int rowNum) + { + return getInt(rowNum); + } + + @Override + public float getFloat(int rowNum) + { + return getInt(rowNum); + } + + @Override + public long getLong(int rowNum) + { + return getInt(rowNum); + } + + @Override + public int getInt(int rowNum) + { + // cell is 0-indexed, rowNumbers are 1-indexed, so add 1. + return rowNum + 1; + } + + @Override + public int compareCells(int lhsRowNum, int rhsRowNum) + { + return Integer.compare(lhsRowNum, rhsRowNum); + } + } + ) + ); + return retVal; + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + return otherProcessor instanceof WindowRowNumberProcessor; + } + + @Override + public String toString() + { + return "WindowRowNumberProcessor{" + + "outputColumn='" + outputColumn + '\'' + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/value/ShiftedColumnAccessorBase.java b/processing/src/main/java/org/apache/druid/query/operator/window/value/ShiftedColumnAccessorBase.java new file mode 100644 index 000000000000..22288a197f2d --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/value/ShiftedColumnAccessorBase.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.segment.column.ColumnType; + +public abstract class ShiftedColumnAccessorBase implements ColumnAccessor +{ + private final ColumnAccessor accessor; + + public ShiftedColumnAccessorBase(ColumnAccessor accessor) + { + this.accessor = accessor; + } + + @Override + public ColumnType getType() + { + return accessor.getType(); + } + + @Override + public int numRows() + { + return accessor.numRows(); + } + + @Override + public boolean isNull(int rowNum) + { + final int actualCell = getActualCell(rowNum); + if (outsideBounds(actualCell)) { + return true; + } + return accessor.isNull(actualCell); + } + + @Override + public Object getObject(int rowNum) + { + final int actualCell = getActualCell(rowNum); + if (outsideBounds(actualCell)) { + return null; + } + return accessor.getObject(actualCell); + } + + @Override + public double getDouble(int rowNum) + { + final int actualCell = getActualCell(rowNum); + if (outsideBounds(actualCell)) { + return 0.0D; + } + return accessor.getDouble(actualCell); + } + + @Override + public float getFloat(int rowNum) + { + final int actualCell = getActualCell(rowNum); + if (outsideBounds(actualCell)) { + return 0.0F; + } + return accessor.getFloat(actualCell); + } + + @Override + public long getLong(int rowNum) + { + final int actualCell = getActualCell(rowNum); + if (outsideBounds(actualCell)) { + return 0L; + } + return accessor.getLong(actualCell); + } + + @Override + public int getInt(int rowNum) + { + final int actualCell = getActualCell(rowNum); + if (outsideBounds(actualCell)) { + return 0; + } + return accessor.getInt(actualCell); + } + + @Override + public int compareCells(int lhsRowNum, int rhsRowNum) + { + int actualLhsCell = getActualCell(lhsRowNum); + int actualRhsCell = getActualCell(rhsRowNum); + if (outsideBounds(actualLhsCell)) { + if (outsideBounds(actualRhsCell)) { + // Both are null + return 0; + } else { + return accessor.isNull(actualRhsCell) ? 0 : -1; + } + } else { + if (outsideBounds(actualRhsCell)) { + return accessor.isNull(actualLhsCell) ? 0 : 1; + } else { + return accessor.compareCells(actualLhsCell, actualRhsCell); + } + } + } + + protected abstract int getActualCell(int cell); + + protected abstract boolean outsideBounds(int cell); +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowFirstProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowFirstProcessor.java new file mode 100644 index 000000000000..fe0cca8884c3 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowFirstProcessor.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.query.rowsandcols.column.ConstantObjectColumn; + +public class WindowFirstProcessor extends WindowValueProcessorBase +{ + @JsonCreator + public WindowFirstProcessor( + @JsonProperty("inputColumn") String inputColumn, + @JsonProperty("outputColumn") String outputColumn + ) + { + super(inputColumn, outputColumn); + } + + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + return processInternal( + incomingPartition, + column -> { + final ColumnAccessor accessor = column.toAccessor(); + return new ConstantObjectColumn(accessor.getObject(0), accessor.numRows(), accessor.getType()); + } + ); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowLastProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowLastProcessor.java new file mode 100644 index 000000000000..2e28962e4346 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowLastProcessor.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.query.rowsandcols.column.ConstantObjectColumn; + +public class WindowLastProcessor extends WindowValueProcessorBase +{ + @JsonCreator + public WindowLastProcessor( + @JsonProperty("inputColumn") String inputColumn, + @JsonProperty("outputColumn") String outputColumn + ) + { + super(inputColumn, outputColumn); + } + + @Override + public RowsAndColumns process(RowsAndColumns input) + { + final int lastIndex = input.numRows() - 1; + if (lastIndex < 0) { + throw new ISE("Called with an input partition of size 0. The call site needs to not do that."); + } + + return processInternal(input, column -> { + final ColumnAccessor accessor = column.toAccessor(); + return new ConstantObjectColumn(accessor.getObject(lastIndex), accessor.numRows(), accessor.getType()); + }); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowOffsetProcessor.java b/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowOffsetProcessor.java new file mode 100644 index 000000000000..4128731be840 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowOffsetProcessor.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.ColumnAccessorBasedColumn; + +public class WindowOffsetProcessor extends WindowValueProcessorBase +{ + private final int offset; + + @JsonCreator + public WindowOffsetProcessor( + @JsonProperty("inputColumn") String inputColumn, + @JsonProperty("outputColumn") String outputColumn, + @JsonProperty("offset") int offset + ) + { + super(inputColumn, outputColumn); + this.offset = offset; + } + + @JsonProperty("offset") + public int getOffset() + { + return offset; + } + + @Override + public RowsAndColumns process(RowsAndColumns input) + { + final int numRows = input.numRows(); + + return processInternal(input, column -> new ColumnAccessorBasedColumn( + new ShiftedColumnAccessorBase(column.toAccessor()) + { + @Override + protected int getActualCell(int cell) + { + return cell + offset; + } + + @Override + protected boolean outsideBounds(int actualCell) + { + return actualCell < 0 || actualCell >= numRows; + } + })); + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + if (otherProcessor instanceof WindowOffsetProcessor) { + WindowOffsetProcessor other = (WindowOffsetProcessor) otherProcessor; + return offset == other.offset && intervalValidation(other); + } + return false; + } + + @Override + public String toString() + { + return "WindowOffsetProcessor{" + + internalToString() + + ", offset=" + offset + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowValueProcessorBase.java b/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowValueProcessorBase.java new file mode 100644 index 000000000000..5486f69b9d3b --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/operator/window/value/WindowValueProcessorBase.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.rowsandcols.AppendableRowsAndColumns; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; + +import java.util.function.Function; + +public abstract class WindowValueProcessorBase implements Processor +{ + private final String inputColumn; + private final String outputColumn; + + public WindowValueProcessorBase( + String inputColumn, + String outputColumn + ) + { + this.inputColumn = inputColumn; + this.outputColumn = outputColumn; + } + + @JsonProperty("inputColumn") + public String getInputColumn() + { + return inputColumn; + } + + @JsonProperty("outputColumn") + public String getOutputColumn() + { + return outputColumn; + } + + /** + * This implements the common logic between the various value processors. It looks like it could be static, but if + * it is static then the lambda becomes polymorphic. We keep it as a member method of the base class so taht the + * JVM can inline it and specialize the lambda + * + * @param input incoming RowsAndColumns, as in Processor.process + * @param fn function that converts the input column into the output column + * @return RowsAndColumns, as in Processor.process + */ + public RowsAndColumns processInternal(RowsAndColumns input, Function fn) + { + final AppendableRowsAndColumns retVal = RowsAndColumns.expectAppendable(input); + + final Column column = input.findColumn(inputColumn); + if (column == null) { + throw new ISE("column[%s] doesn't exist, but window function FIRST wants it to", inputColumn); + } + + retVal.addColumn(outputColumn, fn.apply(column)); + return retVal; + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + return getClass() == otherProcessor.getClass() + && intervalValidation((WindowValueProcessorBase) otherProcessor); + } + + protected boolean intervalValidation(WindowValueProcessorBase other) + { + // Only input needs to be the same for the processors to produce equivalent results + return inputColumn.equals(other.inputColumn); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + internalToString() + '}'; + } + + protected String internalToString() + { + return "inputColumn=" + inputColumn + + ", outputColumn='" + outputColumn + '\''; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/AppendableRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/AppendableRowsAndColumns.java new file mode 100644 index 000000000000..55b197db6122 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/AppendableRowsAndColumns.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.query.rowsandcols.column.Column; + +/** + * A RowsAndColumns that supports appending columns. This interface is particularly useful because even if there is + * some composition of code that works with RowsAndColumns, we would like to add the columns to a singular base object + * instead of build up a complex object graph. + */ +public interface AppendableRowsAndColumns extends RowsAndColumns +{ + /** + * Mutates the RowsAndColumns by appending the requested Column. + * + * @param name the name of the new column + * @param column the Column object representing the new column + */ + void addColumn(String name, Column column); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumns.java new file mode 100644 index 000000000000..3162cdfadbfb --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumns.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.query.rowsandcols.column.ObjectColumnAccessorBase; +import org.apache.druid.segment.RowAdapter; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.Optional; +import java.util.function.Function; + +public class ArrayListRowsAndColumns implements RowsAndColumns +{ + private final ArrayList rows; + private final RowAdapter rowAdapter; + private final RowSignature rowSignature; + + public ArrayListRowsAndColumns( + ArrayList rows, + RowAdapter rowAdapter, + RowSignature rowSignature + ) + { + this.rows = rows; + this.rowAdapter = rowAdapter; + this.rowSignature = rowSignature; + } + + @Override + public Collection getColumnNames() + { + return rowSignature.getColumnNames(); + } + + @Override + public int numRows() + { + return rows.size(); + } + + @Override + @Nullable + public Column findColumn(String name) + { + if (!rowSignature.contains(name)) { + return null; + } + + final Function adapterForValue = rowAdapter.columnFunction(name); + final Optional maybeColumnType = rowSignature.getColumnType(name); + final ColumnType columnType = maybeColumnType.orElse(ColumnType.UNKNOWN_COMPLEX); + final Comparator comparator = Comparator.nullsFirst(columnType.getStrategy()); + + return new Column() + { + @Override + public ColumnAccessor toAccessor() + { + return new ObjectColumnAccessorBase() + { + @Override + protected Object getVal(int cell) + { + return adapterForValue.apply(rows.get(cell)); + } + + @Override + protected Comparator getComparator() + { + return comparator; + } + + @Override + public ColumnType getType() + { + return columnType; + } + + @Override + public int numRows() + { + return rows.size(); + } + }; + } + + @Override + public T as(Class clazz) + { + return null; + } + }; + } + + @Nullable + @Override + public T as(Class clazz) + { + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultGroupPartitioner.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultGroupPartitioner.java new file mode 100644 index 000000000000..96e559e358b2 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultGroupPartitioner.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; + +import java.util.List; + +@SuppressWarnings("unused") +public class DefaultGroupPartitioner implements GroupPartitioner +{ + private final RowsAndColumns rac; + + public DefaultGroupPartitioner( + RowsAndColumns rac + ) + { + this.rac = rac; + } + + @Override + public int[] computeGroupings(List columns) + { + int[] retVal = new int[rac.numRows()]; + + for (String column : columns) { + final Column theCol = rac.findColumn(column); + if (theCol == null) { + // The column doesn't exist. In this case, we assume it's always the same value: null. If it's always + // the same, then it doesn't impact grouping at all and can be entirely skipped. + continue; + } + final ColumnAccessor accessor = theCol.toAccessor(); + + int currGroup = 0; + int prevGroupVal = 0; + for (int i = 1; i < retVal.length; ++i) { + if (retVal[i] == prevGroupVal) { + int comparison = accessor.compareCells(i - 1, i); + if (comparison == 0) { + retVal[i] = currGroup; + continue; + } else if (comparison > 0) { // "greater than" + throw new ISE("Pre-sorted data required, rows[%s] and [%s] were not in order", i - 1, i); + } // the 3rd condition ("less than") means create a new group, so let it fall through + } + + // We have a new group, so walk things forward. + prevGroupVal = retVal[i]; + retVal[i] = ++currGroup; + } + } + + return retVal; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultOnHeapAggregatable.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultOnHeapAggregatable.java new file mode 100644 index 000000000000..83aa50aee8de --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultOnHeapAggregatable.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.aggregation.Aggregator; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.segment.BaseSingleValueDimensionSelector; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.DimensionSelector; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.serde.ComplexMetricSerde; +import org.apache.druid.segment.serde.ComplexMetrics; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +public class DefaultOnHeapAggregatable implements OnHeapAggregatable, OnHeapCumulativeAggregatable +{ + private final RowsAndColumns rac; + + public DefaultOnHeapAggregatable( + RowsAndColumns rac + ) + { + this.rac = rac; + } + + @Override + public ArrayList aggregateAll( + List aggFactories + ) + { + Aggregator[] aggs = new Aggregator[aggFactories.size()]; + + AtomicInteger currRow = new AtomicInteger(0); + int index = 0; + for (AggregatorFactory aggFactory : aggFactories) { + aggs[index++] = aggFactory.factorize(new ColumnAccessorBasedColumnSelectorFactory(currRow)); + } + + int numRows = rac.numRows(); + int rowId = currRow.get(); + while (rowId < numRows) { + for (Aggregator agg : aggs) { + agg.aggregate(); + } + rowId = currRow.incrementAndGet(); + } + + ArrayList retVal = new ArrayList<>(aggs.length); + for (Aggregator agg : aggs) { + retVal.add(agg.get()); + } + return retVal; + } + + @Override + public ArrayList aggregateCumulative(List aggFactories) + { + Aggregator[] aggs = new Aggregator[aggFactories.size()]; + ArrayList retVal = new ArrayList<>(aggFactories.size()); + + int numRows = rac.numRows(); + AtomicInteger currRow = new AtomicInteger(0); + int index = 0; + for (AggregatorFactory aggFactory : aggFactories) { + aggs[index++] = aggFactory.factorize(new ColumnAccessorBasedColumnSelectorFactory(currRow)); + retVal.add(new Object[numRows]); + } + + int rowId = currRow.get(); + while (rowId < numRows) { + for (int i = 0; i < aggs.length; ++i) { + aggs[i].aggregate(); + retVal.get(i)[rowId] = aggs[i].get(); + } + rowId = currRow.incrementAndGet(); + } + + return retVal; + } + + private class ColumnAccessorBasedColumnSelectorFactory implements ColumnSelectorFactory + { + private final Map accessorCache = new HashMap<>(); + + private final AtomicInteger cellIdSupplier; + + public ColumnAccessorBasedColumnSelectorFactory(AtomicInteger cellIdSupplier) + { + this.cellIdSupplier = cellIdSupplier; + } + + @Override + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) + { + return withColumnAccessor(dimensionSpec.getDimension(), columnAccessor -> { + if (columnAccessor == null) { + return DimensionSelector.constant(null); + } else { + return new BaseSingleValueDimensionSelector() + { + @Nullable + @Override + protected String getValue() + { + return String.valueOf(columnAccessor.getObject(cellIdSupplier.get())); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + + } + }; + } + }); + } + + @SuppressWarnings("rawtypes") + @Override + public ColumnValueSelector makeColumnValueSelector(@Nonnull String columnName) + { + return withColumnAccessor(columnName, columnAccessor -> { + if (columnAccessor == null) { + return DimensionSelector.constant(null); + } else { + return new ColumnValueSelector() + { + private final AtomicReference myClazz = new AtomicReference<>(null); + + @Nullable + @Override + public Object getObject() + { + return columnAccessor.getObject(cellIdSupplier.get()); + } + + @SuppressWarnings("rawtypes") + @Override + public Class classOfObject() + { + Class retVal = myClazz.get(); + if (retVal == null) { + retVal = findClazz(); + myClazz.set(retVal); + } + return retVal; + } + + private Class findClazz() + { + final ColumnType type = columnAccessor.getType(); + switch (type.getType()) { + case LONG: + return long.class; + case DOUBLE: + return double.class; + case FLOAT: + return float.class; + case STRING: + return String.class; + case ARRAY: + return List.class; + case COMPLEX: + final ComplexMetricSerde serdeForType = ComplexMetrics.getSerdeForType(type.getComplexTypeName()); + if (serdeForType != null && serdeForType.getObjectStrategy() != null) { + return serdeForType.getObjectStrategy().getClazz(); + } + + for (int i = 0; i < columnAccessor.numRows(); ++i) { + Object obj = columnAccessor.getObject(i); + if (obj != null) { + return obj.getClass(); + } + } + return Object.class; + default: + throw new ISE("Unknown type[%s]", type.getType()); + } + } + + @Override + public boolean isNull() + { + return columnAccessor.isNull(cellIdSupplier.get()); + } + + @Override + public long getLong() + { + return columnAccessor.getLong(cellIdSupplier.get()); + } + + @Override + public float getFloat() + { + return columnAccessor.getFloat(cellIdSupplier.get()); + } + + @Override + public double getDouble() + { + return columnAccessor.getDouble(cellIdSupplier.get()); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + + } + }; + } + }); + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + return withColumnAccessor(column, columnAccessor -> + new ColumnCapabilitiesImpl() + .setType(columnAccessor.getType()) + .setDictionaryEncoded(false) + .setHasBitmapIndexes(false)); + } + + private T withColumnAccessor(String column, Function fn) + { + ColumnAccessor retVal = accessorCache.get(column); + if (retVal == null) { + Column racColumn = rac.findColumn(column); + retVal = racColumn == null ? null : racColumn.toAccessor(); + accessorCache.put(column, retVal); + } + return fn.apply(retVal); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultSortedGroupPartitioner.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultSortedGroupPartitioner.java new file mode 100644 index 000000000000..1470ac2da06d --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/DefaultSortedGroupPartitioner.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.operator.LimitedRowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; + +import java.util.ArrayList; +import java.util.List; + +public class DefaultSortedGroupPartitioner implements SortedGroupPartitioner +{ + private final RowsAndColumns rac; + + public DefaultSortedGroupPartitioner( + RowsAndColumns rac + ) + { + this.rac = rac; + } + + @Override + public int[] computeBoundaries(List columns) + { + // Initialize to a grouping of everything + IntList boundaries = new IntArrayList(new int[]{0, rac.numRows()}); + + for (String column : columns) { + final Column theCol = rac.findColumn(column); + if (theCol == null) { + // The column doesn't exist. In this case, we assume it's always the same value: null. If it's always + // the same, then it doesn't impact grouping at all and can be entirely skipped. + continue; + } + final ColumnAccessor accessor = theCol.toAccessor(); + + IntList newBoundaries = new IntArrayList(); + newBoundaries.add(0); + for (int i = 1; i < boundaries.size(); ++i) { + int start = boundaries.getInt(i - 1); + int end = boundaries.getInt(i); + for (int j = start + 1; j < end; ++j) { + int comparison = accessor.compareCells(j - 1, j); + if (comparison < 0) { + newBoundaries.add(j); + } else if (comparison > 0) { + throw new ISE("Pre-sorted data required, rows[%s] and [%s] were not in order", j - 1, j); + } + } + newBoundaries.add(end); + } + boundaries = newBoundaries; + } + + return boundaries.toIntArray(); + } + + @Override + public ArrayList partitionOnBoundaries(List partitionColumns) + { + final int[] boundaries = computeBoundaries(partitionColumns); + ArrayList retVal = new ArrayList<>(boundaries.length - 1); + + for (int i = 1; i < boundaries.length; ++i) { + int start = boundaries[i - 1]; + int end = boundaries[i]; + retVal.add(new LimitedRowsAndColumns(rac, start, end)); + } + + return retVal; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/GroupPartitioner.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/GroupPartitioner.java new file mode 100644 index 000000000000..e3bae9d4284e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/GroupPartitioner.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import java.util.List; + +/** + * A semantic interface used to partition a data set based on a given set of dimensions. + */ +@SuppressWarnings("unused") +public interface GroupPartitioner +{ + /** + * Computes the groupings of the underlying rows based on the columns passed in for grouping. The grouping is + * returned as an int[], the length of the array will be equal to the number of rows of data and the values of + * the elements of the array will be the same when the rows are part of the same group and different when the + * rows are part of different groups. This is contrasted with the SortedGroupPartitioner in that, the + * groupings returned are not necessarily contiguous. There is also no sort-order implied by the `int` values + * assigned to each grouping. + * + * @param columns the columns to group with + * @return the groupings, rows with the same int value are in the same group. There is no sort-order implied by the + * int values. + */ + int[] computeGroupings(List columns); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/OnHeapAggregatable.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/OnHeapAggregatable.java new file mode 100644 index 000000000000..9a707737a183 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/OnHeapAggregatable.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.query.aggregation.AggregatorFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * A semantic interface used to aggregate a list of AggregatorFactories across a given set of data + *

+ * The aggregation specifically happens on-heap and should be used in places where it is known that the data + * set can be worked with entirely on-heap. + *

+ * Note, as we implement frame-handling for window aggregations, it is expected that this interface will undergo a + * transformation. It might be deleted and replaced with something else, or might just see a change done in place. + * Either way, there is no assumption of enforced compatibility with this interface at this point in time. + */ +public interface OnHeapAggregatable +{ + /** + * Aggregates the data using the {@code List aggregateAll(List aggFactories); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/OnHeapCumulativeAggregatable.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/OnHeapCumulativeAggregatable.java new file mode 100644 index 000000000000..a931c3dbcab1 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/OnHeapCumulativeAggregatable.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.query.aggregation.AggregatorFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * A semantic interface used to cumulatively aggregate a list of AggregatorFactories across a given set of data + *

+ * The aggregation specifically happens on-heap and should be used in places where it is known that the data + * set can be worked with entirely on-heap. + *

+ * Note, as we implement frame-handling for window aggregations, it is expected that this interface will undergo a + * transformation. It might be deleted and replaced with something else, or might just see a change done in place. + * Either way, there is no assumption of enforced compatibility with this interface at this point in time. + */ +public interface OnHeapCumulativeAggregatable +{ + /** + * Cumulatively aggregates the data using the {@code List aggregateCumulative(List aggFactories); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/RowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/RowsAndColumns.java new file mode 100644 index 000000000000..c42f5c0a9267 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/RowsAndColumns.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.frame.AppendableMapOfColumns; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Collection; + +/** + * An interface representing a chunk of RowsAndColumns. Essentially a RowsAndColumns is just a batch of rows + * with columns. + *

+ * This interface has very little prescriptively defined about what *must* be implemented. This is intentional + * as there are lots of different possible representations of batch of rows each with their own unique positives + * and negatives when it comes to processing. So, any explicit definition of what a RowsAndColumns is will actually, + * by definition, end up as optimal for one specific configuration and sub-optimal for others. Instead of trying to + * explicitly expand the interface to cover all the different possible ways that someone could want to interace + * with a Rows and columns, we rely on semantic interfaces using the {@link RowsAndColumns#as} method instead. + *

+ * That is, the expectation is that anything that works with a RowsAndColumns will tend to first ask the RowsAndColumns + * object to become some other interface, for example, an {@link OnHeapAggregatable}. If a RowsAndColumns knows how + * to do a good job as the requested interface, it can return its own concrete implementation of the interface and + * run the necessary logic in its own optimized fashion. If the RowsAndColumns instance does not know how to implement + * the semantic interface, it is expected that a default implementation of the interface can be instantiated on top of + * the default column access mechanisms that the RowsAndColumns provides. Such default implementations should be + * functionally correct, but are not believed to be optimal. + *

+ * The "default column access mechanisms" here amount to using {@link #findColumn} to load a Column + * and then using {@link Column#toAccessor} to access the individual cells of the column. There is also a + * {@link Column#as} method which a default implementation might attempt to use to create a more optimal runtime. + *

+ * It is intended that this interface can be used by Frames, Segments and even normal on-heap JVM data structures to + * participate in query operations. + */ +public interface RowsAndColumns +{ + @Nonnull + static AppendableRowsAndColumns expectAppendable(RowsAndColumns input) + { + if (input instanceof AppendableRowsAndColumns) { + return (AppendableRowsAndColumns) input; + } + + AppendableRowsAndColumns retVal = input.as(AppendableRowsAndColumns.class); + if (retVal == null) { + retVal = new AppendableMapOfColumns(input); + } + return retVal; + } + + /** + * The set of column names available from the RowsAndColumns + * + * @return The set of column names available from the RowsAndColumns + */ + @SuppressWarnings("unreachable") + Collection getColumnNames(); + + /** + * The number of rows in the RowsAndColumns object + * + * @return the integer number of rows + */ + int numRows(); + + /** + * Finds a column by name. null is returned if the column is not found. The RowsAndColumns object should not + * attempt to default not-found columns to pretend as if they exist, instead the user of the RowsAndColumns object + * should decide the correct semantic interpretation of a column that does not exist. It is expected that most + * locations will choose to believe that the column does exist and is always null, but there are often optimizations + * that can effect this same assumption without doing a lot of extra work if the calling code knows that it does not + * exist. + * + * @param name the name of the column to find + * @return the Column, if found. null if not found. + */ + Column findColumn(String name); + + /** + * Asks the RowsAndColumns to return itself as a concrete implementation of a specific interface. The interface + * asked for will tend to be a semantically-meaningful interface. This method allows the calling code to interrogate + * the RowsAndColumns object about whether it can offer a meaningful optimization of the semantic interface. If a + * RowsAndColumns cannot do anything specifically optimal for the interface requested, it should return null instead + * of trying to come up with its own default implementation. + * + * @param clazz A class object representing the interface that the calling code wants a concrete implementation of + * @param The interface that the calling code wants a concrete implementation of + * @return A concrete implementation of the interface, or null if there is no meaningful optimization to be had + * through a local implementation of the interface. + */ + @Nullable + T as(Class clazz); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/SortedGroupPartitioner.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/SortedGroupPartitioner.java new file mode 100644 index 000000000000..a60657937aef --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/SortedGroupPartitioner.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import java.util.ArrayList; +import java.util.List; + +/** + * A semantic interface used to partition a data set based on a given set of dimensions. + *

+ * This specifically assumes that it is working with sorted data and, as such, the groups returned + * should be contiguous and unique (that is, all rows for a given combination of values exist in only one grouping) + */ +public interface SortedGroupPartitioner +{ + /** + * Computes and returns a list of contiguous boundaries for independent groups. All rows in a specific grouping + * should have the same values for the identified columns. Additionally, as this is assuming it is dealing with + * sorted data, there should only be a single entry in the return value for a given set of values of the columns. + * + * @param columns the columns to partition on + * @return an int[] representing the start (inclusive) and stop (exclusive) offsets of boundaries. Boundaries are + * contiguous, so the stop of the previous boundary is the start of the subsequent one. + */ + int[] computeBoundaries(List columns); + + /** + * Semantically equivalent to computeBoundaries, but returns a list of RowsAndColumns objects instead of just + * boundary positions. This is useful as it allows the concrete implementation to return RowsAndColumns objects + * that are aware of the internal representation of the data and thus can provide optimized implementations of + * other semantic interfaces as the "child" RowsAndColumns are used + * + * @param partitionColumns the columns to partition on + * @return a list of RowsAndColumns representing the data grouped by the partition columns. + */ + ArrayList partitionOnBoundaries(List partitionColumns); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/Column.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/Column.java new file mode 100644 index 000000000000..97327faf9ce6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/Column.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import javax.annotation.Nonnull; + +/** + * An interface representing a Column of data. + * + * This interface prescribes that a {@link ColumnAccessor} must be defined on the column, but also offers an + * {@link #as} method to allow for optimized specific implementations of semantically meaningful logic. + * + * That is, the expectation is that some things work with Column objects might choose to first ask the Column + * object to become some other interface. If the Column knows how to do a good job as the requested interface, it can + * return its own concrete implementation of the interface and run the necessary logic in its own optimized fashion. + * If the Column instance does not know how to implement the semantic interface, it is expected that the + * {@link ColumnAccessor} will be leveraged to implement whatever logic is required. + */ +public interface Column +{ + /** + * Returns the column as a {@link ColumnAccessor}. Semantically, this would be equivalent to calling + * {@Code Column.as(ColumnAccessor.class)}. However, being able to implement this interface is part of the explicit + * contract of implementing this interface, so instead of relying on {@link #as} which allows for returning null, + * we define a top-level method that should never return null. + * + * @return a {@link ColumnAccessor} representation of the column, this should never return null. + */ + @Nonnull + ColumnAccessor toAccessor(); + + /** + * Asks the Column to return itself as a concrete implementation of a specific interface. The interface + * asked for will tend to be a semantically-meaningful interface. This method allows the calling code to interrogate + * the Column object about whether it can offer a meaningful optimization of the semantic interface. If a + * Column cannot do anything specifically optimal for the interface requested, it should return null instead + * of trying to come up with its own default implementation. + * + * @param clazz A class object representing the interface that the calling code wants a concrete implementation of + * @param The interface that the calling code wants a concrete implementation of + * @return A concrete implementation of the interface, or null if there is no meaningful optimization to be had + * through a local implementation of the interface. + */ + @SuppressWarnings("unused") + T as(Class clazz); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ColumnAccessor.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ColumnAccessor.java new file mode 100644 index 000000000000..acc6b5806d2f --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ColumnAccessor.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import org.apache.druid.segment.column.ColumnType; + +import javax.annotation.Nullable; + +/** + * Allows for accessing a column, provides methods to enable cell-by-cell access. + */ +public interface ColumnAccessor +{ + /** + * Get the type of the Column + * + * @return the type of the Column + */ + ColumnType getType(); + + /** + * Get the number of cells + * + * @return the number of cells + */ + int numRows(); + + /** + * Get whether the value of a cell is null + * + * @param rowNum the cell id, 0-indexed + * @return true if the value is null + */ + boolean isNull(int rowNum); + + /** + * Get the {@link Object} representation of the cell. + * + * @param rowNum the cell id, 0-indexed + * @return the {@link Object} representation of the cell. Returns {@code null} If {@link #isNull} is true. + */ + @Nullable + Object getObject(int rowNum); + + /** + * Get the primitive {@code double} representation of the cell. + * + * @param rowNum the cell id, 0-indexed + * @return the primitive {@code double} representation of the cell. Returns {@code 0D} If {@link #isNull} is true. + */ + double getDouble(int rowNum); + + /** + * Get the primitive {@code float} representation of the cell. + * + * @param rowNum the cell id, 0-indexed + * @return the primitive {@code float} representation of the cell. Returns {@code 0F} If {@link #isNull} is true. + */ + float getFloat(int rowNum); + + /** + * Get the primitive {@code long} representation of the cell. + * + * @param rowNum the cell id, 0-indexed + * @return the primitive {@code long} representation of the cell. Returns {@code 0L} If {@link #isNull} is true. + */ + long getLong(int rowNum); + + /** + * Get the primitive {@code int} representation of the cell. + * + * @param rowNum the cell id, 0-indexed + * @return the primitive {@code int} representation of the cell. Returns {@code 0} If {@link #isNull} is true. + */ + int getInt(int rowNum); + + /** + * Compares two cells using a comparison that follows the same semantics as {@link java.util.Comparator#compare} + *

+ * This is not comparing the cell Ids, but the values referred to by the cell ids. + * + * @param lhsRowNum the cell id of the left-hand-side of the comparison + * @param rhsRowNum the cell id of the right-hand-side of the comparison + * @return the result of the comparison of the two cells + */ + int compareCells(int lhsRowNum, int rhsRowNum); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ColumnAccessorBasedColumn.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ColumnAccessorBasedColumn.java new file mode 100644 index 000000000000..2e6a4a597cef --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ColumnAccessorBasedColumn.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +public class ColumnAccessorBasedColumn implements Column +{ + private final ColumnAccessor base; + + public ColumnAccessorBasedColumn( + ColumnAccessor base + ) + { + this.base = base; + } + + @Override + public ColumnAccessor toAccessor() + { + return base; + } + + @Override + public T as(Class clazz) + { + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ConstantObjectColumn.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ConstantObjectColumn.java new file mode 100644 index 000000000000..09aad34692dd --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ConstantObjectColumn.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import org.apache.druid.segment.column.ColumnType; + +public class ConstantObjectColumn implements Column +{ + private final Object obj; + private final int numCells; + private final ColumnType type; + + public ConstantObjectColumn(Object obj, int numCells, ColumnType type) + { + this.obj = obj; + this.numCells = numCells; + this.type = type; + } + + @Override + public ColumnAccessor toAccessor() + { + return new ColumnAccessor() + { + @Override + public ColumnType getType() + { + return type; + } + + @Override + public int numRows() + { + return numCells; + } + + @Override + public boolean isNull(int rowNum) + { + return obj == null; + } + + @Override + public Object getObject(int rowNum) + { + return obj; + } + + @Override + public double getDouble(int rowNum) + { + return ((Number) obj).doubleValue(); + } + + @Override + public float getFloat(int rowNum) + { + return ((Number) obj).floatValue(); + } + + @Override + public long getLong(int rowNum) + { + return ((Number) obj).longValue(); + } + + @Override + public int getInt(int rowNum) + { + return ((Number) obj).intValue(); + } + + @Override + public int compareCells(int lhsRowNum, int rhsRowNum) + { + return 0; + } + }; + } + + @Override + public T as(Class clazz) + { + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/DoubleArrayColumn.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/DoubleArrayColumn.java new file mode 100644 index 000000000000..c2d7547be129 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/DoubleArrayColumn.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import org.apache.druid.segment.column.ColumnType; + +public class DoubleArrayColumn implements Column +{ + private final double[] vals; + + public DoubleArrayColumn( + double[] vals + ) + { + this.vals = vals; + } + + @Override + public ColumnAccessor toAccessor() + { + return new ColumnAccessor() + { + @Override + public ColumnType getType() + { + return ColumnType.DOUBLE; + } + + @Override + public int numRows() + { + return vals.length; + } + + @Override + public boolean isNull(int rowNum) + { + return false; + } + + @Override + public Object getObject(int rowNum) + { + return vals[rowNum]; + } + + @Override + public double getDouble(int rowNum) + { + return vals[rowNum]; + } + + @Override + public float getFloat(int rowNum) + { + return (float) vals[rowNum]; + } + + @Override + public long getLong(int rowNum) + { + return (long) vals[rowNum]; + } + + @Override + public int getInt(int rowNum) + { + return (int) vals[rowNum]; + } + + @Override + public int compareCells(int lhsRowNum, int rhsRowNum) + { + return Double.compare(lhsRowNum, rhsRowNum); + } + }; + } + + @Override + public T as(Class clazz) + { + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/IntArrayColumn.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/IntArrayColumn.java new file mode 100644 index 000000000000..10f70351efaa --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/IntArrayColumn.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import org.apache.druid.segment.column.ColumnType; + +public class IntArrayColumn implements Column +{ + private final int[] vals; + + public IntArrayColumn( + int[] vals + ) + { + this.vals = vals; + } + + @Override + public ColumnAccessor toAccessor() + { + return new ColumnAccessor() + { + @Override + public ColumnType getType() + { + return ColumnType.LONG; + } + + @Override + public int numRows() + { + return vals.length; + } + + @Override + public boolean isNull(int rowNum) + { + return false; + } + + @Override + public Object getObject(int rowNum) + { + return vals[rowNum]; + } + + @Override + public double getDouble(int rowNum) + { + return vals[rowNum]; + } + + @Override + public float getFloat(int rowNum) + { + return vals[rowNum]; + } + + @Override + public long getLong(int rowNum) + { + return vals[rowNum]; + } + + @Override + public int getInt(int rowNum) + { + return vals[rowNum]; + } + + @Override + public int compareCells(int lhsRowNum, int rhsRowNum) + { + return Integer.compare(vals[lhsRowNum], vals[rhsRowNum]); + } + }; + } + + @Override + public T as(Class clazz) + { + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/NullColumnAccessor.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/NullColumnAccessor.java new file mode 100644 index 000000000000..ea876c3e1478 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/NullColumnAccessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import org.apache.druid.segment.column.ColumnType; + +import javax.annotation.Nullable; + +public class NullColumnAccessor implements ColumnAccessor +{ + private final ColumnType type; + private final int size; + + public NullColumnAccessor(int size) + { + this(ColumnType.UNKNOWN_COMPLEX, size); + } + + public NullColumnAccessor(ColumnType type, int size) + { + this.type = type; + this.size = size; + } + + @Override + public ColumnType getType() + { + return type; + } + + @Override + public int numRows() + { + return size; + } + + @Override + public boolean isNull(int rowNum) + { + return true; + } + + @Nullable + @Override + public Object getObject(int rowNum) + { + return null; + } + + @Override + public double getDouble(int rowNum) + { + return 0; + } + + @Override + public float getFloat(int rowNum) + { + return 0; + } + + @Override + public long getLong(int rowNum) + { + return 0; + } + + @Override + public int getInt(int rowNum) + { + return 0; + } + + @Override + public int compareCells(int lhsRowNum, int rhsRowNum) + { + return 0; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ObjectArrayColumn.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ObjectArrayColumn.java new file mode 100644 index 000000000000..fd850a0956d0 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ObjectArrayColumn.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import org.apache.druid.segment.column.ColumnType; + +import java.util.Comparator; + +public class ObjectArrayColumn implements Column +{ + private final Object[] objects; + private final ColumnType resultType; + private final Comparator comparator; + + public ObjectArrayColumn(Object[] objects, ColumnType resultType) + { + this(objects, resultType, Comparator.nullsFirst(resultType.getStrategy())); + } + + public ObjectArrayColumn(Object[] objects, ColumnType resultType, Comparator comparator) + { + this.objects = objects; + this.resultType = resultType; + this.comparator = comparator; + } + + @Override + public ColumnAccessor toAccessor() + { + return new ObjectColumnAccessorBase() + { + @Override + protected Object getVal(int cell) + { + return objects[cell]; + } + + @Override + protected Comparator getComparator() + { + return comparator; + } + + @Override + public ColumnType getType() + { + return resultType; + } + + @Override + public int numRows() + { + return objects.length; + } + }; + } + + @Override + public T as(Class clazz) + { + return null; + } + +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ObjectColumnAccessorBase.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ObjectColumnAccessorBase.java new file mode 100644 index 000000000000..8ef2aeae1196 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/column/ObjectColumnAccessorBase.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import javax.annotation.Nullable; +import java.util.Comparator; + +public abstract class ObjectColumnAccessorBase implements ColumnAccessor +{ + @Override + public boolean isNull(int rowNum) + { + return getVal(rowNum) == null; + } + + @Nullable + @Override + public Object getObject(int rowNum) + { + return getVal(rowNum); + } + + @Override + public double getDouble(int rowNum) + { + final Object val = getVal(rowNum); + if (val instanceof Number) { + return ((Number) val).doubleValue(); + } else if (val instanceof String) { + try { + return Double.parseDouble((String) val); + } + catch (NumberFormatException e) { + return 0d; + } + } else { + return 0d; + } + } + + @Override + public float getFloat(int rowNum) + { + final Object val = getVal(rowNum); + if (val instanceof Number) { + return ((Number) val).floatValue(); + } else if (val instanceof String) { + try { + return Float.parseFloat((String) val); + } + catch (NumberFormatException e) { + return 0f; + } + } else { + return 0f; + } + } + + @Override + public long getLong(int rowNum) + { + final Object val = getVal(rowNum); + if (val instanceof Number) { + return ((Number) val).longValue(); + } else if (val instanceof String) { + try { + return Long.parseLong((String) val); + } + catch (NumberFormatException e) { + return 0L; + } + } else { + return 0L; + } + } + + @Override + public int getInt(int rowNum) + { + final Object val = getVal(rowNum); + if (val instanceof Number) { + return ((Number) val).intValue(); + } else if (val instanceof String) { + try { + return Integer.parseInt((String) val); + } + catch (NumberFormatException e) { + return 0; + } + } else { + return 0; + } + } + + @Override + public int compareCells(int lhsRowNum, int rhsRowNum) + { + return getComparator().compare(getVal(lhsRowNum), getVal(rhsRowNum)); + } + + protected abstract Object getVal(int cell); + + protected abstract Comparator getComparator(); +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/frame/AppendableMapOfColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/frame/AppendableMapOfColumns.java new file mode 100644 index 000000000000..60a8d9fa5310 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/frame/AppendableMapOfColumns.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.frame; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.rowsandcols.AppendableRowsAndColumns; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Set; + +public class AppendableMapOfColumns implements AppendableRowsAndColumns +{ + private final RowsAndColumns base; + private final LinkedHashMap appendedColumns; + private Set colNames = null; + + public AppendableMapOfColumns( + RowsAndColumns base + ) + { + this.base = base; + this.appendedColumns = new LinkedHashMap<>(); + } + + @Override + public void addColumn(String name, Column column) + { + final Column prevValue = appendedColumns.put(name, column); + if (prevValue != null) { + throw new ISE("Tried to override column[%s]!? Was[%s], now[%s]", name, prevValue, column); + } + if (colNames != null) { + colNames.add(name); + } + } + + @Override + public Collection getColumnNames() + { + if (colNames == null) { + this.colNames = new LinkedHashSet<>(base.getColumnNames()); + this.colNames.addAll(appendedColumns.keySet()); + } + return colNames; + } + + @Override + public int numRows() + { + return base.numRows(); + } + + @Override + public Column findColumn(String name) + { + Column retVal = base.findColumn(name); + if (retVal == null) { + retVal = appendedColumns.get(name); + } + return retVal; + } + + @Override + @SuppressWarnings("unchecked") + public T as(Class clazz) + { + if (AppendableRowsAndColumns.class.equals(clazz)) { + return (T) this; + } + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/frame/MapOfColumnsRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/frame/MapOfColumnsRowsAndColumns.java new file mode 100644 index 000000000000..aee614b51422 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/frame/MapOfColumnsRowsAndColumns.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.frame; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.rowsandcols.AppendableRowsAndColumns; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; + +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +public class MapOfColumnsRowsAndColumns implements RowsAndColumns +{ + public static MapOfColumnsRowsAndColumns of(String name, Column col) + { + return fromMap(ImmutableMap.of(name, col)); + } + + public static MapOfColumnsRowsAndColumns of(String name, Column col, String name2, Column col2) + { + return fromMap(ImmutableMap.of(name, col, name2, col2)); + } + + public static MapOfColumnsRowsAndColumns fromMap(Map map) + { + if (map == null || map.isEmpty()) { + throw new ISE("map[%s] cannot be null or empty.", map); + } + + final Iterator> iter = map.entrySet().iterator(); + Map.Entry entry = iter.next(); + int numCells = entry.getValue().toAccessor().numRows(); + if (iter.hasNext()) { + entry = iter.next(); + final int newCells = entry.getValue().toAccessor().numRows(); + if (numCells != newCells) { + throw new ISE( + "Mismatched numCells, expectedNumCells[%s], actual[%s] from col[%s].", + numCells, + newCells, + entry.getKey() + ); + } + } + + return new MapOfColumnsRowsAndColumns(map, map.values().iterator().next().toAccessor().numRows()); + } + + private final Map mapOfColumns; + private final int numRows; + + public MapOfColumnsRowsAndColumns( + Map mapOfColumns, + int numRows + ) + { + this.mapOfColumns = mapOfColumns; + this.numRows = numRows; + } + + @Override + public Set getColumnNames() + { + return mapOfColumns.keySet(); + } + + @Override + public int numRows() + { + return numRows; + } + + @Override + public Column findColumn(String name) + { + return mapOfColumns.get(name); + } + + @Override + @SuppressWarnings("unchecked") + public T as(Class clazz) + { + if (AppendableRowsAndColumns.class.equals(clazz)) { + return (T) new AppendableMapOfColumns(this); + } + return null; + } + +} diff --git a/processing/src/main/java/org/apache/druid/segment/ArrayListSegment.java b/processing/src/main/java/org/apache/druid/segment/ArrayListSegment.java new file mode 100644 index 000000000000..97fe80341616 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/ArrayListSegment.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment; + +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.rowsandcols.ArrayListRowsAndColumns; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.timeline.SegmentId; +import org.joda.time.Interval; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; + +/** + * A {@link Segment} that is based on a stream of objects. + */ +public class ArrayListSegment implements Segment +{ + private final SegmentId segmentId; + private final ArrayList rows; + private final RowAdapter rowAdapter; + private final RowSignature rowSignature; + + /** + * Create a list-based segment. + *

+ * The provided List must be in time-order according to the provided {@link RowAdapter#timestampFunction()}. + * The cursor returned by {@link RowBasedStorageAdapter#makeCursors} makes no attempt to verify this, and callers + * will expect it. + *

+ * The provided "rowSignature" will be used for reporting available columns and their capabilities to users of + * {@link #asStorageAdapter()}. Note that the {@link ColumnSelectorFactory} implementation returned by this segment's + * storage adapter will allow creation of selectors on any field, using the {@link RowAdapter#columnFunction} for that + * field, even if it doesn't appear in "rowSignature". + * + * @param segmentId segment identifier; will be returned by {@link #getId()} + * @param rows objects that comprise this segment. Must be re-iterable if support for {@link Cursor#reset()} + * is required. Otherwise, does not need to be re-iterable. + * @param rowAdapter adapter used for reading these objects + * @param rowSignature signature of the columns in these objects + */ + public ArrayListSegment( + final SegmentId segmentId, + final ArrayList rows, + final RowAdapter rowAdapter, + final RowSignature rowSignature + ) + { + this.segmentId = Preconditions.checkNotNull(segmentId, "segmentId"); + this.rows = rows; + this.rowAdapter = rowAdapter; + this.rowSignature = rowSignature; + } + + @Override + @Nonnull + public SegmentId getId() + { + return segmentId; + } + + @Override + @Nonnull + public Interval getDataInterval() + { + return Intervals.ETERNITY; + } + + @Nullable + @Override + public QueryableIndex asQueryableIndex() + { + return null; + } + + @Override + @Nonnull + public StorageAdapter asStorageAdapter() + { + return new RowBasedStorageAdapter<>(Sequences.simple(rows), rowAdapter, rowSignature); + } + + @Nullable + @Override + @SuppressWarnings("unchecked") + public T as(Class clazz) + { + if (RowsAndColumns.class.equals(clazz)) { + return (T) asRowsAndColumns(); + } + return null; + } + + @Override + public void close() + { + // Do nothing. + } + + private RowsAndColumns asRowsAndColumns() + { + return new ArrayListRowsAndColumns(rows, rowAdapter, rowSignature); + } + +} diff --git a/processing/src/main/java/org/apache/druid/segment/BaseSingleValueDimensionSelector.java b/processing/src/main/java/org/apache/druid/segment/BaseSingleValueDimensionSelector.java index adb0ecd1e6e7..c864e6abbc25 100644 --- a/processing/src/main/java/org/apache/druid/segment/BaseSingleValueDimensionSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/BaseSingleValueDimensionSelector.java @@ -50,7 +50,6 @@ public int getValueCardinality() @Override public String lookupName(int id) { - assert id == 0; return getValue(); } diff --git a/processing/src/main/java/org/apache/druid/segment/Segment.java b/processing/src/main/java/org/apache/druid/segment/Segment.java index 245c776b0aca..104216f38c90 100644 --- a/processing/src/main/java/org/apache/druid/segment/Segment.java +++ b/processing/src/main/java/org/apache/druid/segment/Segment.java @@ -23,6 +23,7 @@ import org.apache.druid.timeline.SegmentId; import org.joda.time.Interval; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.Closeable; @@ -57,9 +58,9 @@ public interface Segment extends Closeable * @param desired interface * @return instance of clazz, or null if the interface is not supported by this segment */ - @SuppressWarnings("unused") + @SuppressWarnings({"unused", "unchecked"}) @Nullable - default T as(Class clazz) + default T as(@Nonnull Class clazz) { if (clazz.equals(QueryableIndex.class)) { return (T) asQueryableIndex(); diff --git a/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java index d05ba208585c..93b992a4f428 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java +++ b/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java @@ -82,9 +82,9 @@ public int write(ByteBuffer buffer, T value, int maxSizeBytes) } @Override - public int compare(T o1, T o2) + public int compare(Object o1, Object o2) { - return objectStrategy.compare(o1, o2); + return objectStrategy.compare((T) o1, (T) o2); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java index f1d6c234dcd0..41b84a638e7c 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java @@ -407,9 +407,9 @@ public int write(ByteBuffer buffer, Integer value, int maxSizeBytes) } @Override - public int compare(Integer o1, Integer o2) + public int compare(Object o1, Object o2) { - return Integer.compare(o1, o2); + return Integer.compare(((Number) o1).intValue(), ((Number) o2).intValue()); } } } diff --git a/processing/src/test/java/org/apache/druid/query/operator/InlineScanOperator.java b/processing/src/test/java/org/apache/druid/query/operator/InlineScanOperator.java new file mode 100644 index 000000000000..dbe2adf830d0 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/InlineScanOperator.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; +import org.apache.druid.query.rowsandcols.RowsAndColumns; + +import java.util.Iterator; +import java.util.List; + +public class InlineScanOperator implements Operator +{ + public static InlineScanOperator make(RowsAndColumns item) + { + return new InlineScanOperator(Iterators.singletonIterator(item)); + } + + public static InlineScanOperator make(List items) + { + return new InlineScanOperator(items.iterator()); + } + + private Iterator iter; + + public InlineScanOperator( + Iterator iter + ) + { + Preconditions.checkNotNull(iter); + this.iter = iter; + } + + @Override + public void open() + { + } + + @Override + public RowsAndColumns next() + { + return iter.next(); + } + + @Override + public boolean hasNext() + { + return iter.hasNext(); + } + + @Override + public void close(boolean cascade) + { + iter = null; + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/NaivePartitioningOperatorTest.java b/processing/src/test/java/org/apache/druid/query/operator/NaivePartitioningOperatorTest.java new file mode 100644 index 000000000000..5813f36630d1 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/NaivePartitioningOperatorTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +public class NaivePartitioningOperatorTest +{ + @Test + public void testDefaultImplementation() + { + RowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap( + ImmutableMap.of( + "sorted", new IntArrayColumn(new int[]{0, 0, 0, 1, 1, 2, 4, 4, 4}), + "unsorted", new IntArrayColumn(new int[]{3, 54, 21, 1, 5, 54, 2, 3, 92}) + ) + ); + + NaivePartitioningOperator op = new NaivePartitioningOperator( + ImmutableList.of("sorted"), + InlineScanOperator.make(rac) + ); + + op.open(); + + List expectations = Arrays.asList( + new RowsAndColumnsHelper() + .expectColumn("sorted", new int[]{0, 0, 0}) + .expectColumn("unsorted", new int[]{3, 54, 21}), + new RowsAndColumnsHelper() + .expectColumn("sorted", new int[]{1, 1}) + .expectColumn("unsorted", new int[]{1, 5}), + new RowsAndColumnsHelper() + .expectColumn("sorted", new int[]{2}) + .expectColumn("unsorted", new int[]{54}), + new RowsAndColumnsHelper() + .expectColumn("sorted", new int[]{4, 4, 4}) + .expectColumn("unsorted", new int[]{2, 3, 92}) + ); + + for (RowsAndColumnsHelper expectation : expectations) { + Assert.assertTrue(op.hasNext()); + expectation.validate(op.next()); + } + Assert.assertFalse(op.hasNext()); + + op.close(true); + } + + @Test + public void testFailUnsorted() + { + RowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap( + ImmutableMap.of( + "sorted", new IntArrayColumn(new int[]{0, 0, 0, 1, 1, 2, 4, 4, 4}), + "unsorted", new IntArrayColumn(new int[]{3, 54, 21, 1, 5, 54, 2, 3, 92}) + ) + ); + + NaivePartitioningOperator op = new NaivePartitioningOperator( + ImmutableList.of("unsorted"), + InlineScanOperator.make(rac) + ); + + op.open(); + + boolean exceptionThrown = false; + try { + op.next(); + } + catch (ISE ex) { + Assert.assertEquals("Pre-sorted data required, rows[1] and [2] were not in order", ex.getMessage()); + exceptionThrown = true; + } + Assert.assertTrue(exceptionThrown); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/OperatorSequenceTest.java b/processing/src/test/java/org/apache/druid/query/operator/OperatorSequenceTest.java new file mode 100644 index 000000000000..5d114b8bb2aa --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/OperatorSequenceTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.java.util.common.guava.Yielder; +import org.apache.druid.java.util.common.guava.YieldingAccumulator; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Assert; +import org.junit.Test; + +public class OperatorSequenceTest +{ + @Test + public void testSanity() + { + OperatorSequence seq = new OperatorSequence( + () -> InlineScanOperator.make(MapOfColumnsRowsAndColumns.of("hi", new IntArrayColumn(new int[]{1}))) + ); + + Assert.assertEquals(1, seq.accumulate(0, (accumulated, in) -> accumulated + 1).intValue()); + + Yielder yielder = seq.toYielder(0, new YieldingAccumulator() + { + @Override + public Integer accumulate(Integer accumulated, RowsAndColumns in) + { + yield(); + return accumulated + 1; + } + }); + Assert.assertFalse(yielder.isDone()); + Assert.assertEquals(1, yielder.get().intValue()); + + yielder = yielder.next(0); + Assert.assertTrue(yielder.isDone()); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/SequenceOperatorTest.java b/processing/src/test/java/org/apache/druid/query/operator/SequenceOperatorTest.java new file mode 100644 index 000000000000..09f52562d4b9 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/SequenceOperatorTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +public class SequenceOperatorTest +{ + @Test + public void testSanity() + { + SequenceOperator op = new SequenceOperator(Sequences.simple(Arrays.asList( + MapOfColumnsRowsAndColumns.of("hi", new IntArrayColumn(new int[]{1})), + MapOfColumnsRowsAndColumns.of("hi", new IntArrayColumn(new int[]{1})) + ))); + + op.open(); + + RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("hi", new int[]{1}) + .allColumnsRegistered(); + + expectations.validate(op.next()); + Assert.assertTrue(op.hasNext()); + + expectations.validate(op.next()); + Assert.assertFalse(op.hasNext()); + + op.close(true); + op.close(false); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/WindowOperatorQueryTest.java b/processing/src/test/java/org/apache/druid/query/operator/WindowOperatorQueryTest.java new file mode 100644 index 000000000000..dc2e6e9732d4 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/WindowOperatorQueryTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.QueryContext; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.segment.column.RowSignature; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Set; + +/** + * Tests the WindowOperatorQuery, it would actually be a lot better to run this through some tests that actually + * validate the operation of queries, but all of the efforts to build out test scaffolding and framework have gone + * into building things out for SQL query operations. As such, all of the tests that validating the actual native + * functionality actually run from the `druid-sql` module instead of this module. It would be best to de-couple + * these and have all of the native, query processing tests happen directly here in processing and have the SQL + * tests only concern themselves with how they plan SQL into Native, but that's a bit big of a nugget to bite off + * at this point in time, so instead we continue the building of technical debt by making this "test" run lines + * of code without actually testing much meaningful behavior. + *

+ * For now, view CalciteWindowQueryTest for actual tests that validate behavior. + */ +public class WindowOperatorQueryTest +{ + WindowOperatorQuery query; + + @Before + public void setUp() + { + query = new WindowOperatorQuery( + InlineDataSource.fromIterable(new ArrayList<>(), RowSignature.empty()), + ImmutableMap.of("sally", "sue"), + RowSignature.empty(), + new ArrayList<>() + ); + } + + @Test + public void getOperators() + { + Assert.assertTrue(query.getOperators().isEmpty()); + } + + @Test + public void getRowSignature() + { + Assert.assertEquals(0, query.getRowSignature().size()); + } + + @Test + public void hasFilters() + { + Assert.assertFalse(query.hasFilters()); + } + + @Test + public void getFilter() + { + Assert.assertNull(query.getFilter()); + } + + @Test + public void getType() + { + Assert.assertEquals("windowOperator", query.getType()); + } + + @Test + public void withOverriddenContext() + { + Assert.assertEquals("sue", query.context().get("sally")); + final QueryContext context = query.withOverriddenContext(ImmutableMap.of("sally", "soo")).context(); + Assert.assertEquals("soo", context.get("sally")); + } + + @Test + public void withDataSource() + { + final Set tableNames = query.getDataSource().getTableNames(); + Assert.assertEquals(0, tableNames.size()); + + boolean exceptionThrown = false; + try { + query.withDataSource(new TableDataSource("bob")); + } + catch (IAE e) { + // should fail trying to set a TableDataSource as TableDataSource is not currently allowed. + exceptionThrown = true; + } + Assert.assertTrue(exceptionThrown); + } + + @Test + public void testEquals() + { + Assert.assertEquals(query, query); + Assert.assertEquals(query, query.withDataSource(query.getDataSource())); + Assert.assertNotEquals(query, query.toString()); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/WindowProcessorOperatorTest.java b/processing/src/test/java/org/apache/druid/query/operator/WindowProcessorOperatorTest.java new file mode 100644 index 000000000000..39ba5996d5d4 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/WindowProcessorOperatorTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Assert; +import org.junit.Test; + +public class WindowProcessorOperatorTest +{ + @Test + public void testJustRunsTheProcessor() + { + RowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap( + ImmutableMap.of( + "colA", new IntArrayColumn(new int[]{1, 2, 3}), + "colB", new IntArrayColumn(new int[]{3, 2, 1}) + ) + ); + + WindowProcessorOperator op = new WindowProcessorOperator( + new Processor() + { + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + return incomingPartition; + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + return true; + } + }, + InlineScanOperator.make(rac) + ); + + op.open(); + Assert.assertTrue(op.hasNext()); + Assert.assertSame(rac, op.next()); + Assert.assertFalse(op.hasNext()); + op.close(true); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/ComposingProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/ComposingProcessorTest.java new file mode 100644 index 000000000000..570cba65d92c --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/ComposingProcessorTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window; + +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.junit.Assert; +import org.junit.Test; + +public class ComposingProcessorTest +{ + @Test + public void testSanity() + { + final ProcessorForTesting firstProcessor = new ProcessorForTesting(); + final ProcessorForTesting secondProcessor = new ProcessorForTesting(); + + ComposingProcessor proc = new ComposingProcessor(firstProcessor, secondProcessor); + + proc.process(null); + Assert.assertEquals(1, firstProcessor.processCounter); + Assert.assertEquals(1, secondProcessor.processCounter); + + proc.process(null); + Assert.assertEquals(2, firstProcessor.processCounter); + Assert.assertEquals(2, secondProcessor.processCounter); + + Assert.assertTrue(proc.validateEquivalent(proc)); + Assert.assertEquals(1, firstProcessor.validateCounter); + Assert.assertEquals(1, secondProcessor.validateCounter); + + firstProcessor.validationResult = false; + Assert.assertFalse(proc.validateEquivalent(proc)); + Assert.assertEquals(2, firstProcessor.validateCounter); + Assert.assertEquals(1, secondProcessor.validateCounter); + } + + private static class ProcessorForTesting implements Processor + { + private int processCounter = 0; + private int validateCounter = 0; + private boolean validationResult = true; + + @Override + public RowsAndColumns process(RowsAndColumns incomingPartition) + { + ++processCounter; + return incomingPartition; + } + + @Override + public boolean validateEquivalent(Processor otherProcessor) + { + ++validateCounter; + return validationResult; + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/RowsAndColumnsHelper.java b/processing/src/test/java/org/apache/druid/query/operator/window/RowsAndColumnsHelper.java new file mode 100644 index 000000000000..451ead6a9396 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/RowsAndColumnsHelper.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window; + +import com.google.common.collect.ImmutableSet; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.segment.column.ColumnType; +import org.junit.Assert; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +public class RowsAndColumnsHelper +{ + public static void assertEquals(RowsAndColumns rac, String name, int[] expectedResults) + { + final Column column = rac.findColumn(name); + Assert.assertNotNull(column); + final ColumnAccessor accessor = column.toAccessor(); + Assert.assertEquals(expectedResults.length, accessor.numRows()); + for (int i = 0; i < expectedResults.length; ++i) { + Assert.assertEquals(StringUtils.format("%s[%s]", name, i), expectedResults[i], accessor.getInt(i)); + } + } + + public static void assertEquals(RowsAndColumns rac, String name, long[] expectedResults) + { + final Column column = rac.findColumn(name); + Assert.assertNotNull(column); + final ColumnAccessor accessor = column.toAccessor(); + Assert.assertEquals(expectedResults.length, accessor.numRows()); + for (int i = 0; i < expectedResults.length; ++i) { + Assert.assertEquals(StringUtils.format("%s[%s]", name, i), expectedResults[i], accessor.getLong(i)); + } + } + + public static void assertEquals(RowsAndColumns rac, String name, double[] expectedResults) + { + final Column column = rac.findColumn(name); + Assert.assertNotNull(column); + final ColumnAccessor accessor = column.toAccessor(); + Assert.assertEquals(expectedResults.length, accessor.numRows()); + for (int i = 0; i < expectedResults.length; ++i) { + Assert.assertEquals(StringUtils.format("%s[%s]", name, i), expectedResults[i], accessor.getDouble(i), 0.0d); + } + } + + private final Map helpers = new LinkedHashMap<>(); + private Set fullColumnSet; + + public RowsAndColumnsHelper() + { + } + + public RowsAndColumnsHelper expectColumn(String col, int[] expectedVals) + { + final ColumnHelper helper = columnHelper(col, expectedVals.length, ColumnType.LONG); + helper.setExpectation(expectedVals); + return this; + } + + public RowsAndColumnsHelper expectColumn(String col, long[] expectedVals) + { + final ColumnHelper helper = columnHelper(col, expectedVals.length, ColumnType.LONG); + helper.setExpectation(expectedVals); + return this; + } + + public RowsAndColumnsHelper expectColumn(String col, double[] expectedVals) + { + final ColumnHelper helper = columnHelper(col, expectedVals.length, ColumnType.DOUBLE); + helper.setExpectation(expectedVals); + return this; + } + + public ColumnHelper columnHelper(String column, int expectedSize, ColumnType expectedType) + { + ColumnHelper retVal = helpers.get(column); + if (retVal == null) { + retVal = new ColumnHelper(expectedSize, expectedType); + helpers.put(column, retVal); + return retVal; + } else { + throw new ISE( + "column[%s] expectations already defined, size[%s], type[%s]", + column, + retVal.expectedVals.length, + retVal.expectedType + ); + } + } + + public RowsAndColumnsHelper expectFullColumns(Set fullColumnSet) + { + this.fullColumnSet = fullColumnSet; + return this; + } + + public RowsAndColumnsHelper allColumnsRegistered() + { + this.fullColumnSet = ImmutableSet.copyOf(helpers.keySet()); + return this; + } + + public void validate(RowsAndColumns rac) + { + validate("", rac); + } + + public void validate(String name, RowsAndColumns rac) + { + if (fullColumnSet != null) { + final Collection columnNames = rac.getColumnNames(); + Assert.assertEquals(name, fullColumnSet.size(), columnNames.size()); + Assert.assertTrue(name, fullColumnSet.containsAll(columnNames)); + } + + for (Map.Entry entry : helpers.entrySet()) { + entry.getValue().validate(StringUtils.format("%s.%s", name, entry.getKey()), rac.findColumn(entry.getKey())); + } + } + + public static class ColumnHelper + { + private final ColumnType expectedType; + private final Object[] expectedVals; + private final boolean[] expectedNulls; + + public ColumnHelper(int expectedSize, ColumnType expectedType) + { + this.expectedType = expectedType; + this.expectedVals = new Object[expectedSize]; + this.expectedNulls = new boolean[expectedVals.length]; + } + + public ColumnHelper setExpectation(int[] expectedVals) + { + for (int i = 0; i < expectedVals.length; i++) { + this.expectedVals[i] = expectedVals[i]; + } + return this; + } + + public ColumnHelper setExpectation(long[] expectedVals) + { + for (int i = 0; i < expectedVals.length; i++) { + this.expectedVals[i] = expectedVals[i]; + } + return this; + } + + public ColumnHelper setExpectation(double[] expectedVals) + { + for (int i = 0; i < expectedVals.length; i++) { + this.expectedVals[i] = expectedVals[i]; + } + return this; + } + + public ColumnHelper setExpectation(float[] expectedVals) + { + for (int i = 0; i < expectedVals.length; i++) { + this.expectedVals[i] = expectedVals[i]; + } + return this; + } + + public ColumnHelper setExpectation(Object[] expectedVals) + { + System.arraycopy(expectedVals, 0, this.expectedVals, 0, expectedVals.length); + return this; + } + + public ColumnHelper setNulls(int[] nullIndexes) + { + for (int nullIndex : nullIndexes) { + this.expectedNulls[nullIndex] = true; + } + return this; + } + + public void validate(String msgBase, Column col) + { + final ColumnAccessor accessor = col.toAccessor(); + + Assert.assertEquals(msgBase, expectedType, accessor.getType()); + Assert.assertEquals(msgBase, expectedVals.length, accessor.numRows()); + for (int i = 0; i < accessor.numRows(); ++i) { + final String msg = StringUtils.format("%s[%s]", msgBase, i); + Object expectedVal = expectedVals[i]; + if (expectedVal == null) { + Assert.assertTrue(msg, expectedNulls[i]); + Assert.assertTrue(msg, accessor.isNull(i)); + Assert.assertNull(msg, accessor.getObject(i)); + } + if (expectedVal instanceof Float) { + if (expectedNulls[i]) { + Assert.assertTrue(msg, accessor.isNull(i)); + Assert.assertEquals(msg, 0.0f, accessor.getFloat(i), 0.0); + } else { + Assert.assertFalse(msg, accessor.isNull(i)); + Assert.assertEquals(msg, (Float) expectedVal, accessor.getFloat(i), 0.0); + } + } else if (expectedVal instanceof Double) { + if (expectedNulls[i]) { + Assert.assertTrue(msg, accessor.isNull(i)); + Assert.assertEquals(msg, 0.0d, accessor.getDouble(i), 0.0); + } else { + Assert.assertFalse(msg, accessor.isNull(i)); + Assert.assertEquals(msg, (Double) expectedVal, accessor.getDouble(i), 0.0); + } + } else if (expectedVal instanceof Integer) { + if (expectedNulls[i]) { + Assert.assertTrue(msg, accessor.isNull(i)); + Assert.assertEquals(msg, 0, accessor.getInt(i)); + } else { + Assert.assertFalse(msg, accessor.isNull(i)); + Assert.assertEquals(msg, ((Integer) expectedVal).intValue(), accessor.getInt(i)); + } + } else if (expectedVal instanceof Long) { + if (expectedNulls[i]) { + Assert.assertTrue(msg, accessor.isNull(i)); + Assert.assertEquals(msg, 0, accessor.getLong(i)); + } else { + Assert.assertFalse(msg, accessor.isNull(i)); + Assert.assertEquals(msg, ((Long) expectedVal).longValue(), accessor.getLong(i)); + } + } else { + if (expectedNulls[i]) { + Assert.assertTrue(msg, accessor.isNull(i)); + Assert.assertNull(msg, accessor.getObject(i)); + // asserting null on the expected value is here for consistency in the tests. If it fails, it's most + // likely indicative of something wrong with the test setup than the actual logic, we keep it for + // sanity's sake to things consistent. + Assert.assertNull(msg, expectedVals[i]); + } else { + final Object obj = accessor.getObject(i); + Assert.assertFalse(msg, accessor.isNull(i)); + Assert.assertNotNull(msg, obj); + Assert.assertEquals(msg, expectedVals[i], obj); + } + } + } + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/WindowAggregateProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/WindowAggregateProcessorTest.java new file mode 100644 index 000000000000..856dd0a30421 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/WindowAggregateProcessorTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory; +import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; +import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.operator.window.ranking.WindowRowNumberProcessor; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.apache.druid.segment.column.ColumnType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowAggregateProcessorTest +{ + static { + NullHandling.initializeForTests(); + } + + @Test + public void testAggregation() + { + Map map = new LinkedHashMap<>(); + map.put("intCol", new IntArrayColumn(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("doubleCol", new DoubleArrayColumn(new double[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("objectCol", new ObjectArrayColumn( + new String[]{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + ColumnType.STRING + ) + ); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + WindowAggregateProcessor processor = new WindowAggregateProcessor( + Arrays.asList( + new LongSumAggregatorFactory("sumFromLong", "intCol"), + new LongSumAggregatorFactory("sumFromDouble", "doubleCol"), + new DoubleMaxAggregatorFactory("maxFromInt", "intCol"), + new DoubleMaxAggregatorFactory("maxFromDouble", "doubleCol") + ), + Arrays.asList( + new LongMaxAggregatorFactory("cummMax", "intCol"), + new DoubleSumAggregatorFactory("cummSum", "doubleCol") + ) + ); + + RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("intCol", new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("doubleCol", new double[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("sumFromLong", new long[]{45, 45, 45, 45, 45, 45, 45, 45, 45, 45}) + .expectColumn("sumFromDouble", new long[]{45, 45, 45, 45, 45, 45, 45, 45, 45, 45}) + .expectColumn("maxFromInt", new double[]{9, 9, 9, 9, 9, 9, 9, 9, 9, 9}) + .expectColumn("maxFromDouble", new double[]{9, 9, 9, 9, 9, 9, 9, 9, 9, 9}) + .expectColumn("cummMax", new long[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("cummSum", new double[]{0, 1, 3, 6, 10, 15, 21, 28, 36, 45}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + } + + @Test + public void testValidateEquality() + { + WindowAggregateProcessor processor = new WindowAggregateProcessor( + Arrays.asList( + new LongSumAggregatorFactory("sumFromLong", "intCol"), + new LongSumAggregatorFactory("sumFromDouble", "doubleCol"), + new DoubleMaxAggregatorFactory("maxFromInt", "intCol"), + new DoubleMaxAggregatorFactory("maxFromDouble", "doubleCol") + ), + Arrays.asList( + new LongMaxAggregatorFactory("cummMax", "intCol"), + new DoubleSumAggregatorFactory("cummSum", "doubleCol") + ) + ); + + Assert.assertTrue(processor.validateEquivalent(processor)); + Assert.assertFalse(processor.validateEquivalent(new WindowRowNumberProcessor("bob"))); + Assert.assertFalse(processor.validateEquivalent(new WindowAggregateProcessor(processor.getAggregations(), null))); + Assert.assertFalse(processor.validateEquivalent( + new WindowAggregateProcessor(new ArrayList<>(), processor.getCumulativeAggregations()) + )); + Assert.assertFalse(processor.validateEquivalent(new WindowAggregateProcessor(new ArrayList<>(), null))); + } + +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowCumeDistProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowCumeDistProcessorTest.java new file mode 100644 index 000000000000..7b0bf1448170 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowCumeDistProcessorTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowCumeDistProcessorTest +{ + @Test + public void testCumeDistProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("vals", new IntArrayColumn(new int[]{7, 18, 18, 30, 120, 121, 122, 122, 8290, 8290})); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + Processor processor = new WindowCumeDistProcessor(Collections.singletonList("vals"), "CumeDist"); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("vals", new int[]{7, 18, 18, 30, 120, 121, 122, 122, 8290, 8290}) + .expectColumn("CumeDist", new double[]{0.1, 0.3, 0.3, 0.4, 0.5, 0.6, 0.8, 0.8, 1.0, 1.0}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowDenseRankProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowDenseRankProcessorTest.java new file mode 100644 index 000000000000..d61c40ca48e2 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowDenseRankProcessorTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowDenseRankProcessorTest +{ + @Test + public void testDenseRankProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("vals", new IntArrayColumn(new int[]{7, 18, 18, 30, 120, 121, 122, 122, 8290, 8290})); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + Processor processor = new WindowDenseRankProcessor(Collections.singletonList("vals"), "DenseRank"); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("vals", new int[]{7, 18, 18, 30, 120, 121, 122, 122, 8290, 8290}) + .expectColumn("DenseRank", new int[]{1, 2, 2, 3, 4, 5, 6, 6, 7, 7}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowPercentileProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowPercentileProcessorTest.java new file mode 100644 index 000000000000..cc6473f10a97 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowPercentileProcessorTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import org.apache.druid.query.operator.window.ComposingProcessor; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.apache.druid.segment.column.ColumnType; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowPercentileProcessorTest +{ + @Test + public void testPercentileProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("intCol", new IntArrayColumn(new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("doubleCol", new DoubleArrayColumn(new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("objectCol", new ObjectArrayColumn( + new String[]{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + ColumnType.STRING + )); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + Processor processor = new ComposingProcessor( + new WindowPercentileProcessor("1", 1), + new WindowPercentileProcessor("2", 2), + new WindowPercentileProcessor("3", 3), + new WindowPercentileProcessor("4", 4), + new WindowPercentileProcessor("5", 5), + new WindowPercentileProcessor("6", 6), + new WindowPercentileProcessor("7", 7), + new WindowPercentileProcessor("8", 8), + new WindowPercentileProcessor("9", 9), + new WindowPercentileProcessor("10", 10), + new WindowPercentileProcessor("10292", 10292) + ); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("intCol", new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("doubleCol", new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("1", new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) + .expectColumn("2", new int[]{1, 1, 1, 1, 1, 2, 2, 2, 2, 2}) + .expectColumn("3", new int[]{1, 1, 1, 1, 2, 2, 2, 3, 3, 3}) + .expectColumn("4", new int[]{1, 1, 1, 2, 2, 2, 3, 3, 4, 4}) + .expectColumn("5", new int[]{1, 1, 2, 2, 3, 3, 4, 4, 5, 5}) + .expectColumn("6", new int[]{1, 1, 2, 2, 3, 3, 4, 4, 5, 6}) + .expectColumn("7", new int[]{1, 1, 2, 2, 3, 3, 4, 5, 6, 7}) + .expectColumn("8", new int[]{1, 1, 2, 2, 3, 4, 5, 6, 7, 8}) + .expectColumn("9", new int[]{1, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("10", new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + .expectColumn("10292", new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowRankProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowRankProcessorTest.java new file mode 100644 index 000000000000..3f004a6fd26b --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowRankProcessorTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import org.apache.druid.query.operator.window.ComposingProcessor; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowRankProcessorTest +{ + @Test + public void testRankProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("vals", new IntArrayColumn(new int[]{7, 18, 18, 30, 120, 121, 122, 122, 8290, 8290})); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + Processor processor = new ComposingProcessor( + new WindowRankProcessor(Collections.singletonList("vals"), "rank", false), + new WindowRankProcessor(Collections.singletonList("vals"), "rankAsPercent", true) + ); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("vals", new int[]{7, 18, 18, 30, 120, 121, 122, 122, 8290, 8290}) + .expectColumn("rank", new int[]{1, 2, 2, 4, 5, 6, 7, 7, 9, 9}) + .expectColumn( + "rankAsPercent", + new double[]{0.0, 1 / 9d, 1 / 9d, 3 / 9d, 4 / 9d, 5 / 9d, 6 / 9d, 6 / 9d, 8 / 9d, 8 / 9d} + ); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + + } + + @Test + public void testRankSingle() + { + Map map = new LinkedHashMap<>(); + map.put("vals", new IntArrayColumn(new int[]{7})); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + Processor processor = new ComposingProcessor( + new WindowRankProcessor(Collections.singletonList("vals"), "rank", false), + new WindowRankProcessor(Collections.singletonList("vals"), "rankAsPercent", true) + ); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("vals", new int[]{7}) + .expectColumn("rank", new int[]{1}) + .expectColumn("rankAsPercent", new double[]{0.0}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowRowNumberProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowRowNumberProcessorTest.java new file mode 100644 index 000000000000..bc06de60e880 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/ranking/WindowRowNumberProcessorTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.ranking; + +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.apache.druid.segment.column.ColumnType; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowRowNumberProcessorTest +{ + @Test + public void testRowNumberProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("intCol", new IntArrayColumn(new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("doubleCol", new DoubleArrayColumn(new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("objectCol", new ObjectArrayColumn( + new String[]{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + ColumnType.STRING + )); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + Processor processor = new WindowRowNumberProcessor("rowRow"); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("intCol", new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("doubleCol", new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("rowRow", new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowFirstProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowFirstProcessorTest.java new file mode 100644 index 000000000000..425ff9036012 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowFirstProcessorTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import org.apache.druid.query.operator.window.ComposingProcessor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.apache.druid.segment.column.ColumnType; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowFirstProcessorTest +{ + @Test + public void testFirstProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("intCol", new IntArrayColumn(new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("doubleCol", new DoubleArrayColumn(new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("objectCol", new ObjectArrayColumn( + new String[]{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + ColumnType.STRING + )); + map.put("nullFirstCol", new ObjectArrayColumn( + new String[]{null, "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + ColumnType.STRING + )); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + ComposingProcessor processor = new ComposingProcessor( + new WindowFirstProcessor("intCol", "FirstIntCol"), + new WindowFirstProcessor("doubleCol", "FirstDoubleCol"), + new WindowFirstProcessor("objectCol", "FirstObjectCol"), + new WindowFirstProcessor("nullFirstCol", "NullFirstCol") + ); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("intCol", new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("doubleCol", new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("FirstIntCol", new int[]{88, 88, 88, 88, 88, 88, 88, 88, 88, 88}) + .expectColumn( + "FirstDoubleCol", + new double[]{0.4728, 0.4728, 0.4728, 0.4728, 0.4728, 0.4728, 0.4728, 0.4728, 0.4728, 0.4728} + ); + + expectations.columnHelper("FirstObjectCol", 10, ColumnType.STRING) + .setExpectation(new String[]{"a", "a", "a", "a", "a", "a", "a", "a", "a", "a"}); + + expectations.columnHelper("NullFirstCol", 10, ColumnType.STRING) + .setNulls(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLagProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLagProcessorTest.java new file mode 100644 index 000000000000..3079a6acea80 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLagProcessorTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import org.apache.druid.query.operator.window.ComposingProcessor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.apache.druid.segment.column.ColumnType; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowLagProcessorTest +{ + @Test + public void testLagProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("intCol", new IntArrayColumn(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("doubleCol", new DoubleArrayColumn(new double[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("objectCol", new ObjectArrayColumn( + new String[]{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + ColumnType.STRING + ) + ); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + ComposingProcessor processor = new ComposingProcessor( + new WindowOffsetProcessor("intCol", "laggardIntCol", -2), + new WindowOffsetProcessor("doubleCol", "laggardDoubleCol", -4), + new WindowOffsetProcessor("objectCol", "laggardObjectCol", -1) + ); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("intCol", new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("doubleCol", new double[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + expectations.columnHelper("laggardIntCol", 10, ColumnType.LONG) + .setExpectation(new int[]{0, 0, 0, 1, 2, 3, 4, 5, 6, 7}) + .setNulls(new int[]{0, 1}); + + expectations.columnHelper("laggardDoubleCol", 10, ColumnType.DOUBLE) + .setExpectation(new double[]{0, 0, 0, 0, 0, 1, 2, 3, 4, 5}) + .setNulls(new int[]{0, 1, 2, 3}); + + expectations.columnHelper("laggardObjectCol", 10, ColumnType.STRING) + .setExpectation(new String[]{null, "a", "b", "c", "d", "e", "f", "g", "h", "i"}) + .setNulls(new int[]{0}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLastProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLastProcessorTest.java new file mode 100644 index 000000000000..2e6aabba4971 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLastProcessorTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import org.apache.druid.query.operator.window.ComposingProcessor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.apache.druid.segment.column.ColumnType; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowLastProcessorTest +{ + @Test + public void testLastProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("intCol", new IntArrayColumn(new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("doubleCol", new DoubleArrayColumn(new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9.84})); + map.put("objectCol", new ObjectArrayColumn( + new String[]{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + ColumnType.STRING + )); + map.put("nullLastCol", new ObjectArrayColumn( + new String[]{null, "b", "c", "d", "e", "f", "g", "h", "i", null}, + ColumnType.STRING + )); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + ComposingProcessor processor = new ComposingProcessor( + new WindowLastProcessor("intCol", "LastIntCol"), + new WindowLastProcessor("doubleCol", "LastDoubleCol"), + new WindowLastProcessor("objectCol", "LastObjectCol"), + new WindowLastProcessor("nullLastCol", "NullLastCol") + ); + + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("intCol", new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("doubleCol", new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9.84}) + .expectColumn("LastIntCol", new int[]{9, 9, 9, 9, 9, 9, 9, 9, 9, 9}) + .expectColumn("LastDoubleCol", new double[]{9.84, 9.84, 9.84, 9.84, 9.84, 9.84, 9.84, 9.84, 9.84, 9.84}); + + expectations.columnHelper("LastObjectCol", 10, ColumnType.STRING) + .setExpectation(new String[]{"j", "j", "j", "j", "j", "j", "j", "j", "j", "j"}); + + expectations.columnHelper("NullLastCol", 10, ColumnType.STRING) + .setNulls(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + final RowsAndColumns results = processor.process(rac); + expectations.validate(results); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLeadProcessorTest.java b/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLeadProcessorTest.java new file mode 100644 index 000000000000..fac4bf5e081d --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/operator/window/value/WindowLeadProcessorTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.operator.window.value; + +import org.apache.druid.query.operator.window.ComposingProcessor; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.DoubleArrayColumn; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.apache.druid.segment.column.ColumnType; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class WindowLeadProcessorTest +{ + @Test + public void testLeadProcessing() + { + Map map = new LinkedHashMap<>(); + map.put("intCol", new IntArrayColumn(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("doubleCol", new DoubleArrayColumn(new double[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + map.put("objectCol", new ObjectArrayColumn( + new String[]{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + ColumnType.STRING + ) + ); + + MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(map); + + ComposingProcessor processor = new ComposingProcessor( + new WindowOffsetProcessor("intCol", "LeadingIntCol", 2), + new WindowOffsetProcessor("doubleCol", "LeadingDoubleCol", 4), + new WindowOffsetProcessor("objectCol", "LeadingObjectCol", 1) + ); + + final RowsAndColumns results = processor.process(rac); + + final RowsAndColumnsHelper expectations = new RowsAndColumnsHelper() + .expectColumn("intCol", new int[]{88, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + .expectColumn("doubleCol", new double[]{0.4728, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + expectations.columnHelper("LeadingIntCol", 10, ColumnType.LONG) + .setExpectation(new int[]{2, 3, 4, 5, 6, 7, 8, 9, 0, 0}) + .setNulls(new int[]{8, 9}); + + expectations.columnHelper("LeadingDoubleCol", 10, ColumnType.DOUBLE) + .setExpectation(new double[]{4, 5, 6, 7, 8, 9, 0, 0, 0, 0}) + .setNulls(new int[]{6, 7, 8, 9}); + + expectations.columnHelper("LeadingObjectCol", 10, ColumnType.STRING) + .setExpectation(new String[]{"b", "c", "d", "e", "f", "g", "h", "i", "j", null}) + .setNulls(new int[]{9}); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumnsTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumnsTest.java new file mode 100644 index 000000000000..9d2d18870f5a --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumnsTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.apache.druid.segment.column.RowSignature; + +import java.util.ArrayList; + +public class ArrayListRowsAndColumnsTest extends RowsAndColumnsTestBase> +{ + + @Override + public ArrayListRowsAndColumns makeRowsAndColumns(MapOfColumnsRowsAndColumns input) + { + ArrayList rows = new ArrayList<>(input.numRows()); + + ArrayList cols = new ArrayList<>(input.getColumnNames()); + final RowSignature.Builder sigBob = RowSignature.builder(); + + for (int i = 0; i < input.numRows(); ++i) { + rows.add(new Object[cols.size()]); + } + + for (int colIndex = 0; colIndex < cols.size(); ++colIndex) { + String col = cols.get(colIndex); + final ColumnAccessor column = input.findColumn(col).toAccessor(); + sigBob.add(col, column.getType()); + + for (int i = 0; i < column.numRows(); ++i) { + rows.get(i)[colIndex] = column.getObject(i); + } + } + + return new ArrayListRowsAndColumns<>( + rows, + columnName -> { + final int i = cols.indexOf(columnName); + if (i < 0) { + throw new ISE("Couldn't find column[%s]!? i[%s]", columnName, i); + } + return objects -> objects[i]; + }, + sigBob.build() + ); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/MapOfColumnsRowsAndColumnsTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/MapOfColumnsRowsAndColumnsTest.java new file mode 100644 index 000000000000..4e85c152f853 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/MapOfColumnsRowsAndColumnsTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +public class MapOfColumnsRowsAndColumnsTest extends RowsAndColumnsTestBase +{ + @Override + public MapOfColumnsRowsAndColumns makeRowsAndColumns(MapOfColumnsRowsAndColumns input) + { + return input; + } + + @Test + public void testMakeWithEmptyAndNull() + { + boolean exceptionThrown = false; + try { + MapOfColumnsRowsAndColumns.fromMap(null); + } + catch (ISE ex) { + Assert.assertEquals("map[null] cannot be null or empty.", ex.getMessage()); + exceptionThrown = true; + } + Assert.assertTrue(exceptionThrown); + + exceptionThrown = false; + try { + MapOfColumnsRowsAndColumns.fromMap(Collections.emptyMap()); + } + catch (ISE ex) { + Assert.assertEquals("map[{}] cannot be null or empty.", ex.getMessage()); + exceptionThrown = true; + } + Assert.assertTrue(exceptionThrown); + } + + @Test + public void testExceptionOnMismatchedCells() + { + boolean exceptionThrown = false; + try { + MapOfColumnsRowsAndColumns.of( + "1", new IntArrayColumn(new int[]{0}), + "2", new IntArrayColumn(new int[]{0, 1}) + ); + } + catch (ISE ex) { + Assert.assertEquals("Mismatched numCells, expectedNumCells[1], actual[2] from col[2].", ex.getMessage()); + exceptionThrown = true; + } + Assert.assertTrue(exceptionThrown); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/RowsAndColumnsTestBase.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/RowsAndColumnsTestBase.java new file mode 100644 index 000000000000..d0a4c30f0e6e --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/RowsAndColumnsTestBase.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; +import org.apache.druid.query.aggregation.LongMinAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.operator.window.RowsAndColumnsHelper; +import org.apache.druid.query.rowsandcols.column.IntArrayColumn; +import org.apache.druid.query.rowsandcols.frame.AppendableMapOfColumns; +import org.apache.druid.query.rowsandcols.frame.MapOfColumnsRowsAndColumns; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * This base class is intended to serve as a common set of tests to validate specific RowsAndColumns implementations. + *

+ * Different RowsAndColumns implementations will implement different of the semantic interfaces, this base class should + * test all of the possible semantic interfaces that can be implemented. By doing it this way, we can ensure that + * new RowsAndColumns implementations meet all of the corners cases and other issues that have been previously found. + *

+ * It is expected that this base class is going to grow quite large. As it gets extra large, we could perhaps look + * into whether one of the JUnit test runners could allow us to further sub-divide the test functionality into + * semantic-interface-specific tests. The ultimate goal, however, should be that a new RowsAndColumns implementation + * can very simply take advantage of all of the tests by implementing the abstract + * {@link #makeRowsAndColumns(MapOfColumnsRowsAndColumns)} method and be done. + * + * @param + */ +public abstract class RowsAndColumnsTestBase +{ + static { + NullHandling.initializeForTests(); + } + + public abstract T makeRowsAndColumns(MapOfColumnsRowsAndColumns input); + + @Test + public void testDefaultSortedGroupPartitioner() + { + T rac = makeRowsAndColumns(MapOfColumnsRowsAndColumns.fromMap( + ImmutableMap.of( + "sorted", new IntArrayColumn(new int[]{0, 0, 0, 1, 1, 2, 4, 4, 4}), + "unsorted", new IntArrayColumn(new int[]{3, 54, 21, 1, 5, 54, 2, 3, 92}) + ) + )); + + validateSortedGroupPartitioner("default", new DefaultSortedGroupPartitioner(rac)); + + SortedGroupPartitioner specialized = rac.as(SortedGroupPartitioner.class); + if (specialized != null) { + validateSortedGroupPartitioner("specialized", specialized); + } + } + + private void validateSortedGroupPartitioner(String name, SortedGroupPartitioner parter) + { + + int[] expectedBounds = new int[]{0, 3, 5, 6, 9}; + + List expectations = Arrays.asList( + new RowsAndColumnsHelper() + .expectColumn("sorted", new int[]{0, 0, 0}) + .expectColumn("unsorted", new int[]{3, 54, 21}) + .allColumnsRegistered(), + new RowsAndColumnsHelper() + .expectColumn("sorted", new int[]{1, 1}) + .expectColumn("unsorted", new int[]{1, 5}) + .allColumnsRegistered(), + new RowsAndColumnsHelper() + .expectColumn("sorted", new int[]{2}) + .expectColumn("unsorted", new int[]{54}) + .allColumnsRegistered(), + new RowsAndColumnsHelper() + .expectColumn("sorted", new int[]{4, 4, 4}) + .expectColumn("unsorted", new int[]{2, 3, 92}) + .allColumnsRegistered() + ); + + final List partCols = Collections.singletonList("sorted"); + Assert.assertArrayEquals(name, expectedBounds, parter.computeBoundaries(partCols)); + + final Iterator partedChunks = parter.partitionOnBoundaries(partCols).iterator(); + for (RowsAndColumnsHelper expectation : expectations) { + Assert.assertTrue(name, partedChunks.hasNext()); + expectation.validate(name, partedChunks.next()); + } + Assert.assertFalse(name, partedChunks.hasNext()); + + boolean exceptionThrown = false; + try { + parter.partitionOnBoundaries(Collections.singletonList("unsorted")); + } + catch (ISE ex) { + Assert.assertEquals("Pre-sorted data required, rows[1] and [2] were not in order", ex.getMessage()); + exceptionThrown = true; + } + Assert.assertTrue(exceptionThrown); + } + + @Test + public void testOnHeapAggregatable() + { + T rac = makeRowsAndColumns(MapOfColumnsRowsAndColumns.fromMap( + ImmutableMap.of( + "incremented", new IntArrayColumn(new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}), + "zeroesOut", new IntArrayColumn(new int[]{4, -4, 3, -3, 4, 82, -90, 4, 0, 0}) + ) + )); + + validateOnHeapAggregatable("default", new DefaultOnHeapAggregatable(rac)); + + OnHeapAggregatable specialized = rac.as(OnHeapAggregatable.class); + if (specialized != null) { + validateOnHeapAggregatable("specialized", specialized); + } + } + + private void validateOnHeapAggregatable(String name, OnHeapAggregatable agger) + { + final ArrayList results = agger.aggregateAll(Arrays.asList( + new LongSumAggregatorFactory("incremented", "incremented"), + new LongMaxAggregatorFactory("zeroesOutMax", "zeroesOut"), + new LongMinAggregatorFactory("zeroesOutMin", "zeroesOut") + )); + + Assert.assertEquals(name, 3, results.size()); + Assert.assertEquals(name, 55L, results.get(0)); + Assert.assertEquals(name, 82L, results.get(1)); + Assert.assertEquals(name, -90L, results.get(2)); + } + + @Test + public void testAppendableRowsAndColumns() + { + T rac = makeRowsAndColumns(MapOfColumnsRowsAndColumns.fromMap( + ImmutableMap.of( + "colA", new IntArrayColumn(new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}), + "colB", new IntArrayColumn(new int[]{4, -4, 3, -3, 4, 82, -90, 4, 0, 0}) + ) + )); + + validateAppendableRowsAndColumns("default", new AppendableMapOfColumns(rac)); + + AppendableRowsAndColumns specialized = rac.as(AppendableRowsAndColumns.class); + if (specialized != null) { + validateAppendableRowsAndColumns("specialized", specialized); + } + } + + public void validateAppendableRowsAndColumns(String name, AppendableRowsAndColumns appender) + { + appender.addColumn("newCol", new IntArrayColumn(new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})); + + new RowsAndColumnsHelper() + .expectColumn("colA", new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + .expectColumn("colB", new int[]{4, -4, 3, -3, 4, 82, -90, 4, 0, 0}) + .expectColumn("newCol", new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + .allColumnsRegistered() + .validate(name, appender); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/column/NullColumnAccessorTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/column/NullColumnAccessorTest.java new file mode 100644 index 000000000000..89c286165dc8 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/column/NullColumnAccessorTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.rowsandcols.column; + +import org.junit.Assert; +import org.junit.Test; + +public class NullColumnAccessorTest +{ + + @Test + public void testSanity() + { + NullColumnAccessor accessor = new NullColumnAccessor(10); + Assert.assertEquals(10, accessor.numRows()); + + for (int i = 0; i < 10; ++i) { + Assert.assertTrue(accessor.isNull(i)); + Assert.assertNull(accessor.getObject(i)); + Assert.assertEquals(0, accessor.getInt(i)); + Assert.assertEquals(0, accessor.getLong(i)); + Assert.assertEquals(0.0, accessor.getFloat(i), 0); + Assert.assertEquals(0.0, accessor.getDouble(i), 0); + for (int j = 0; j < i; ++j) { + Assert.assertEquals(0, accessor.compareCells(j, i)); + } + } + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java index 98dc97731f5c..b0396c9293f3 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java @@ -357,7 +357,7 @@ public int write(ByteBuffer buffer, int offset, T value, int maxSizeBytes) } @Override - public int compare(T o1, T o2) + public int compare(Object o1, Object o2) { return delegate.compare(o1, o2); } diff --git a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java index 102c72d890cc..a2fe8f8d2a58 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java @@ -35,6 +35,8 @@ import org.apache.druid.query.groupby.GroupByQueryRunnerFactory; import org.apache.druid.query.metadata.SegmentMetadataQueryRunnerFactory; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.operator.WindowOperatorQuery; +import org.apache.druid.query.operator.WindowOperatorQueryQueryRunnerFactory; import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.scan.ScanQueryRunnerFactory; import org.apache.druid.query.search.SearchQuery; @@ -56,14 +58,15 @@ public class QueryRunnerFactoryModule extends QueryToolChestModule { private static final Map>, Class>> MAPPINGS = ImmutableMap.>, Class>>builder() - .put(TimeseriesQuery.class, TimeseriesQueryRunnerFactory.class) - .put(SearchQuery.class, SearchQueryRunnerFactory.class) - .put(TimeBoundaryQuery.class, TimeBoundaryQueryRunnerFactory.class) - .put(SegmentMetadataQuery.class, SegmentMetadataQueryRunnerFactory.class) + .put(DataSourceMetadataQuery.class, DataSourceMetadataQueryRunnerFactory.class) .put(GroupByQuery.class, GroupByQueryRunnerFactory.class) .put(ScanQuery.class, ScanQueryRunnerFactory.class) + .put(SearchQuery.class, SearchQueryRunnerFactory.class) + .put(SegmentMetadataQuery.class, SegmentMetadataQueryRunnerFactory.class) + .put(TimeBoundaryQuery.class, TimeBoundaryQueryRunnerFactory.class) + .put(TimeseriesQuery.class, TimeseriesQueryRunnerFactory.class) .put(TopNQuery.class, TopNQueryRunnerFactory.class) - .put(DataSourceMetadataQuery.class, DataSourceMetadataQueryRunnerFactory.class) + .put(WindowOperatorQuery.class, WindowOperatorQueryQueryRunnerFactory.class) .build(); @Override diff --git a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java index edb892b03a95..da9ab41e299c 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java @@ -41,6 +41,8 @@ import org.apache.druid.query.metadata.SegmentMetadataQueryConfig; import org.apache.druid.query.metadata.SegmentMetadataQueryQueryToolChest; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.operator.WindowOperatorQuery; +import org.apache.druid.query.operator.WindowOperatorQueryQueryToolChest; import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.scan.ScanQueryConfig; import org.apache.druid.query.scan.ScanQueryQueryToolChest; @@ -75,14 +77,15 @@ public class QueryToolChestModule implements Module public final Map, Class> mappings = ImmutableMap., Class>builder() - .put(TimeseriesQuery.class, TimeseriesQueryQueryToolChest.class) - .put(SearchQuery.class, SearchQueryQueryToolChest.class) - .put(TimeBoundaryQuery.class, TimeBoundaryQueryQueryToolChest.class) - .put(SegmentMetadataQuery.class, SegmentMetadataQueryQueryToolChest.class) + .put(DataSourceMetadataQuery.class, DataSourceQueryQueryToolChest.class) .put(GroupByQuery.class, GroupByQueryQueryToolChest.class) .put(ScanQuery.class, ScanQueryQueryToolChest.class) + .put(SearchQuery.class, SearchQueryQueryToolChest.class) + .put(SegmentMetadataQuery.class, SegmentMetadataQueryQueryToolChest.class) + .put(TimeBoundaryQuery.class, TimeBoundaryQueryQueryToolChest.class) + .put(TimeseriesQuery.class, TimeseriesQueryQueryToolChest.class) .put(TopNQuery.class, TopNQueryQueryToolChest.class) - .put(DataSourceMetadataQuery.class, DataSourceQueryQueryToolChest.class) + .put(WindowOperatorQuery.class, WindowOperatorQueryQueryToolChest.class) .build(); @Override diff --git a/server/src/main/java/org/apache/druid/segment/InlineSegmentWrangler.java b/server/src/main/java/org/apache/druid/segment/InlineSegmentWrangler.java index a68859359cde..6128dcf49f26 100644 --- a/server/src/main/java/org/apache/druid/segment/InlineSegmentWrangler.java +++ b/server/src/main/java/org/apache/druid/segment/InlineSegmentWrangler.java @@ -22,14 +22,14 @@ import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.DataSource; import org.apache.druid.query.InlineDataSource; -import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.timeline.SegmentId; import org.joda.time.Interval; +import java.util.ArrayList; import java.util.Collections; /** - * A {@link JoinableFactory} for {@link InlineDataSource}. + * A {@link SegmentWrangler} for {@link InlineDataSource}. * * It is not valid to pass any other DataSource type to the "getSegmentsForIntervals" method. */ @@ -38,10 +38,22 @@ public class InlineSegmentWrangler implements SegmentWrangler private static final String SEGMENT_ID = "inline"; @Override + @SuppressWarnings("unchecked") public Iterable getSegmentsForIntervals(final DataSource dataSource, final Iterable intervals) { final InlineDataSource inlineDataSource = (InlineDataSource) dataSource; + if (inlineDataSource.rowsAreArrayList()) { + return Collections.singletonList( + new ArrayListSegment<>( + SegmentId.dummy(SEGMENT_ID), + (ArrayList) inlineDataSource.getRowsAsList(), + inlineDataSource.rowAdapter(), + inlineDataSource.getRowSignature() + ) + ); + } + return Collections.singletonList( new RowBasedSegment<>( SegmentId.dummy(SEGMENT_ID), diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index d6fe03a31d63..638e22919201 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -72,9 +72,9 @@ /** * Query handler for the Broker processes (see CliBroker). - * + *

* This class is responsible for: - * + *

* 1) Running queries on the cluster using its 'clusterClient' * 2) Running queries locally (when all datasources are global) using its 'localClient' * 3) Inlining subqueries if necessary, in service of the above two goals @@ -133,7 +133,7 @@ public ClientQuerySegmentWalker( { this( emitter, - (QuerySegmentWalker) clusterClient, + clusterClient, (QuerySegmentWalker) localClient, warehouse, joinableFactory, @@ -291,20 +291,20 @@ private DataSource globalizeIfPossible( /** * Replace QueryDataSources with InlineDataSources when necessary and possible. "Necessary" is defined as: - * + *

* 1) For outermost subqueries: inlining is necessary if the toolchest cannot handle it. * 2) For all other subqueries (e.g. those nested under a join): inlining is always necessary. * - * @param dataSource datasource to process. - * @param toolChestIfOutermost if provided, and if the provided datasource is a {@link QueryDataSource}, this method - * will consider whether the toolchest can handle a subquery on the datasource using - * {@link QueryToolChest#canPerformSubquery}. If the toolchest can handle it, then it will - * not be inlined. See {@link org.apache.druid.query.groupby.GroupByQueryQueryToolChest} - * for an example of a toolchest that can handle subqueries. + * @param dataSource datasource to process. + * @param toolChestIfOutermost if provided, and if the provided datasource is a {@link QueryDataSource}, this method + * will consider whether the toolchest can handle a subquery on the datasource using + * {@link QueryToolChest#canPerformSubquery}. If the toolchest can handle it, then it will + * not be inlined. See {@link org.apache.druid.query.groupby.GroupByQueryQueryToolChest} + * for an example of a toolchest that can handle subqueries. * @param subqueryRowLimitAccumulator an accumulator for tracking the number of accumulated rows in all subqueries * for a particular master query - * @param maxSubqueryRows Max rows that all the subqueries generated by a master query can have, combined - * @param dryRun if true, does not actually execute any subqueries, but will inline empty result sets. + * @param maxSubqueryRows Max rows that all the subqueries generated by a master query can have, combined + * @param dryRun if true, does not actually execute any subqueries, but will inline empty result sets. */ @SuppressWarnings({"rawtypes", "unchecked"}) // Subquery, toolchest, runner handling all use raw types private DataSource inlineIfNecessary( @@ -331,15 +331,18 @@ private DataSource inlineIfNecessary( current = Iterables.getOnlyElement(current.getChildren()); } - assert !(current instanceof QueryDataSource); // lgtm [java/contradictory-type-checks] + if (current instanceof QueryDataSource) { + throw new ISE("Got a QueryDataSource[%s], should've walked it away in the loop above.", current); + } current = inlineIfNecessary(current, null, subqueryRowLimitAccumulator, maxSubqueryRows, dryRun); while (!stack.isEmpty()) { current = stack.pop().withChildren(Collections.singletonList(current)); } - assert current instanceof QueryDataSource; - + if (!(current instanceof QueryDataSource)) { + throw new ISE("Should have a QueryDataSource, but got[%s] instead", current); + } if (toolChest.canPerformSubquery(((QueryDataSource) current).getQuery())) { return current; } else { @@ -431,7 +434,9 @@ private QueryRunner decorateClusterRunner(Query query, QueryRunner .postProcess( objectMapper.convertValue( query.context().getString("postProcessing"), - new TypeReference>() {} + new TypeReference>() + { + } ) ) .map( @@ -572,7 +577,7 @@ private static > InlineDataSource toInlineDataSour final RowSignature signature = toolChest.resultArraySignature(query); - final List resultList = new ArrayList<>(); + final ArrayList resultList = new ArrayList<>(); toolChest.resultsAsArrays(query, results).accumulate( resultList, diff --git a/server/src/main/java/org/apache/druid/server/LocalQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/LocalQuerySegmentWalker.java index 730eb3121d47..767e5fbd402b 100644 --- a/server/src/main/java/org/apache/druid/server/LocalQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/LocalQuerySegmentWalker.java @@ -93,12 +93,10 @@ public QueryRunner getQueryRunnerForIntervals(final Query query, final final AtomicLong cpuAccumulator = new AtomicLong(0L); - final Function segmentMapFn = analysis - .getDataSource() - .createSegmentMapFunction( - query, - cpuAccumulator - ); + final Function segmentMapFn = + analysis + .getDataSource() + .createSegmentMapFunction(query, cpuAccumulator); final QueryRunnerFactory> queryRunnerFactory = conglomerate.findFactory(query); diff --git a/server/src/main/java/org/apache/druid/server/coordination/ServerManager.java b/server/src/main/java/org/apache/druid/server/coordination/ServerManager.java index 26aabc2ddd68..7990a44b8506 100644 --- a/server/src/main/java/org/apache/druid/server/coordination/ServerManager.java +++ b/server/src/main/java/org/apache/druid/server/coordination/ServerManager.java @@ -194,9 +194,11 @@ public QueryRunner getQueryRunnerForSegments(Query query, Iterable(Lists.newArrayList(specs)); } - final Function segmentMapFn = query.getDataSource().createSegmentMapFunction(query, cpuTimeAccumulator); + final Function segmentMapFn = + query.getDataSource() + .createSegmentMapFunction(query, cpuTimeAccumulator); - // We compute the join cache key here itself so it doesn't need to be re-computed for every segment + // We compute the datasource's cache key here itself so it doesn't need to be re-computed for every segment final Optional cacheKeyPrefix = Optional.ofNullable(query.getDataSource().getCacheKey()); final FunctionalIterable> queryRunners = FunctionalIterable diff --git a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java index 5e0701680cd0..967d9342d78f 100644 --- a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java +++ b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java @@ -1391,7 +1391,6 @@ public QueryRunner getQueryRunnerForSegments(Query query, Iterable> timelines, - JoinableFactoryWrapper joinableFactoryWraper, QueryRunnerFactoryConglomerate conglomerate, @Nullable QueryScheduler scheduler ) { - return new TestClusterQuerySegmentWalker(timelines, joinableFactoryWraper, conglomerate, scheduler); + return new TestClusterQuerySegmentWalker(timelines, conglomerate, scheduler); } public static LocalQuerySegmentWalker createLocalQuerySegmentWalker( @@ -348,6 +349,7 @@ public int getMinTopNThreshold() ) .put(GroupByQuery.class, groupByQueryRunnerFactory) .put(TimeBoundaryQuery.class, new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER)) + .put(WindowOperatorQuery.class, new WindowOperatorQueryQueryRunnerFactory()) .build() ); diff --git a/server/src/test/java/org/apache/druid/server/TestClusterQuerySegmentWalker.java b/server/src/test/java/org/apache/druid/server/TestClusterQuerySegmentWalker.java index 2245b28169a8..000626f16e2c 100644 --- a/server/src/test/java/org/apache/druid/server/TestClusterQuerySegmentWalker.java +++ b/server/src/test/java/org/apache/druid/server/TestClusterQuerySegmentWalker.java @@ -45,7 +45,6 @@ import org.apache.druid.query.spec.SpecificSegmentSpec; import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.SegmentReference; -import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.timeline.TimelineObjectHolder; import org.apache.druid.timeline.VersionedIntervalTimeline; import org.apache.druid.timeline.partition.PartitionChunk; @@ -71,20 +70,17 @@ public class TestClusterQuerySegmentWalker implements QuerySegmentWalker { private final Map> timelines; - private final JoinableFactoryWrapper joinableFactoryWrapper; private final QueryRunnerFactoryConglomerate conglomerate; @Nullable private final QueryScheduler scheduler; TestClusterQuerySegmentWalker( Map> timelines, - JoinableFactoryWrapper joinableFactoryWrapper, QueryRunnerFactoryConglomerate conglomerate, @Nullable QueryScheduler scheduler ) { this.timelines = timelines; - this.joinableFactoryWrapper = joinableFactoryWrapper; this.conglomerate = conglomerate; this.scheduler = scheduler; } diff --git a/sql/pom.xml b/sql/pom.xml index 9d260eddf4f7..963ae0287522 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -18,7 +18,8 @@ ~ under the License. --> - + 4.0.0 druid-sql @@ -186,6 +187,7 @@ provided + junit @@ -208,6 +210,11 @@ easymock test + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + test + commons-io commons-io @@ -249,11 +256,11 @@ mockito-core test - - org.jdbi - jdbi - test - + + org.jdbi + jdbi + test + diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java index fa09d99d1f06..b1805519ade9 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java @@ -93,7 +93,12 @@ public static RexNode fromFieldAccess( ) { if (project == null) { - // I don't think the factory impl matters here. + // Gian doesn't think the factory impl matters here, he's likely correct. But, upon reading what this is doing, + // we are re-building the list of things in the RelDataType for every single call to `fromFieldAccess`. + // `fromFieldAccess` is called pretty regularly in pretty low-level areas of the code, so it would make sense + // that we are perhaps re-creating the exact same object over and over and over and over again and wasting CPU + // cycles. It would likely be good to refactor the code such that we ensure we only ever compute the thing + // once and then reuse it. return RexInputRef.of(fieldNumber, RowSignatures.toRelDataType(rowSignature, new JavaTypeFactoryImpl())); } else { return project.getChildExps().get(fieldNumber); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/WindowSqlAggregate.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/WindowSqlAggregate.java new file mode 100644 index 000000000000..7dd158d91f3a --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/WindowSqlAggregate.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.expression; + +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.druid.java.util.common.UOE; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.sql.calcite.aggregation.Aggregation; +import org.apache.druid.sql.calcite.aggregation.SqlAggregator; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; + +import javax.annotation.Nullable; +import java.util.List; + +/** + * Conversion for SQL operators that map 1-1 onto native functions. + */ +public class WindowSqlAggregate implements SqlAggregator +{ + private final SqlAggFunction operator; + + public WindowSqlAggregate(final SqlAggFunction operator) + { + this.operator = operator; + } + + @Override + public SqlAggFunction calciteFunction() + { + return operator; + } + + @Nullable + @Override + public Aggregation toDruidAggregation( + PlannerContext plannerContext, + RowSignature rowSignature, + VirtualColumnRegistry virtualColumnRegistry, + RexBuilder rexBuilder, + String name, + AggregateCall aggregateCall, + Project project, + List existingAggregations, + boolean finalizeAggregations + ) + { + throw new UOE("%s can only be used in a window function, this method shouldn't be called...", operator.getName()); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java index 088a9cb861b1..e2b91f7d79a8 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java @@ -206,6 +206,7 @@ public class CalciteRulesManager /** * Manages the rules for planning of SQL queries via Calcite. Also provides methods for extensions to provide custom * rules for planning. + * * @param extensionCalciteRuleProviderSet the set of custom rules coming from extensions */ @Inject @@ -265,11 +266,11 @@ public List druidConventionRuleSet(final PlannerContext plannerConte public List bindableConventionRuleSet(final PlannerContext plannerContext) { return ImmutableList.builder() - .addAll(baseRuleSet(plannerContext)) - .addAll(Bindables.RULES) - .addAll(DEFAULT_BINDABLE_RULES) - .add(AggregateReduceFunctionsRule.INSTANCE) - .build(); + .addAll(baseRuleSet(plannerContext)) + .addAll(Bindables.RULES) + .addAll(DEFAULT_BINDABLE_RULES) + .add(AggregateReduceFunctionsRule.INSTANCE) + .build(); } public List baseRuleSet(final PlannerContext plannerContext) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java index 2431fc4accb5..26bd698e17c6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java @@ -54,6 +54,7 @@ import org.apache.druid.sql.calcite.expression.UnaryFunctionOperatorConversion; import org.apache.druid.sql.calcite.expression.UnaryPrefixOperatorConversion; import org.apache.druid.sql.calcite.expression.UnarySuffixOperatorConversion; +import org.apache.druid.sql.calcite.expression.WindowSqlAggregate; import org.apache.druid.sql.calcite.expression.builtin.ArrayAppendOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ArrayConcatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ArrayConstructorOperatorConversion; @@ -135,6 +136,16 @@ public class DruidOperatorTable implements SqlOperatorTable // COUNT and APPROX_COUNT_DISTINCT are not here because they are added by SqlAggregationModule. private static final List STANDARD_AGGREGATORS = ImmutableList.builder() + .add(new WindowSqlAggregate(SqlStdOperatorTable.LAG)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.LEAD)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.FIRST_VALUE)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.LAST_VALUE)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.CUME_DIST)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.DENSE_RANK)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.NTILE)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.PERCENT_RANK)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.RANK)) + .add(new WindowSqlAggregate(SqlStdOperatorTable.ROW_NUMBER)) .add(new BuiltinApproxCountDistinctSqlAggregator()) .add(new AvgSqlAggregator()) .add(EarliestLatestAnySqlAggregator.EARLIEST) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index 8c5aa588f708..f4601c03b37c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -49,6 +49,7 @@ import org.apache.druid.query.DataSource; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.Query; +import org.apache.druid.query.QueryContext; import org.apache.druid.query.QueryDataSource; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; @@ -61,6 +62,7 @@ import org.apache.druid.query.groupby.having.DimFilterHavingSpec; import org.apache.druid.query.groupby.orderby.DefaultLimitSpec; import org.apache.druid.query.groupby.orderby.OrderByColumnSpec; +import org.apache.druid.query.operator.WindowOperatorQuery; import org.apache.druid.query.ordering.StringComparator; import org.apache.druid.query.ordering.StringComparators; import org.apache.druid.query.planning.DataSourceAnalysis; @@ -97,7 +99,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -124,10 +125,11 @@ public class DruidQuery /** * Maximum number of time-granular buckets that we allow for non-Druid tables. - * + *

* Used by {@link #canUseQueryGranularity}. */ private static final int MAX_TIME_GRAINS_NON_DRUID_TABLE = 100000; + public static final String CTX_ENABLE_WINDOW_FNS = "windowsAreForClosers"; private final DataSource dataSource; private final PlannerContext plannerContext; @@ -144,6 +146,9 @@ public class DruidQuery @Nullable private final Sorting sorting; + @Nullable + private final Windowing windowing; + private final Query query; private final RowSignature outputRowSignature; private final RelDataType outputRowType; @@ -157,6 +162,7 @@ private DruidQuery( @Nullable final Projection selectProjection, @Nullable final Grouping grouping, @Nullable final Sorting sorting, + @Nullable final Windowing windowing, final RowSignature sourceRowSignature, final RelDataType outputRowType, final VirtualColumnRegistry virtualColumnRegistry @@ -168,9 +174,16 @@ private DruidQuery( this.selectProjection = selectProjection; this.grouping = grouping; this.sorting = sorting; + this.windowing = windowing; this.sourceRowSignature = sourceRowSignature; - this.outputRowSignature = computeOutputRowSignature(sourceRowSignature, selectProjection, grouping, sorting); + this.outputRowSignature = computeOutputRowSignature( + sourceRowSignature, + selectProjection, + grouping, + sorting, + windowing + ); this.outputRowType = Preconditions.checkNotNull(outputRowType, "outputRowType"); this.virtualColumnRegistry = Preconditions.checkNotNull(virtualColumnRegistry, "virtualColumnRegistry"); this.query = computeQuery(); @@ -200,6 +213,7 @@ public static DruidQuery fromPartialQuery( final Projection selectProjection; final Grouping grouping; final Sorting sorting; + final Windowing windowing; if (partialQuery.getWhereFilter() != null) { filter = Preconditions.checkNotNull( @@ -221,7 +235,7 @@ public static DruidQuery fromPartialQuery( computeSelectProjection( partialQuery, plannerContext, - computeOutputRowSignature(sourceRowSignature, null, null, null), + computeOutputRowSignature(sourceRowSignature, null, null, null, null), virtualColumnRegistry ) ); @@ -234,7 +248,7 @@ public static DruidQuery fromPartialQuery( computeGrouping( partialQuery, plannerContext, - computeOutputRowSignature(sourceRowSignature, null, null, null), + computeOutputRowSignature(sourceRowSignature, null, null, null, null), virtualColumnRegistry, rexBuilder, finalizeAggregations @@ -249,7 +263,7 @@ public static DruidQuery fromPartialQuery( computeSorting( partialQuery, plannerContext, - computeOutputRowSignature(sourceRowSignature, selectProjection, grouping, null), + computeOutputRowSignature(sourceRowSignature, selectProjection, grouping, null, null), // When sorting follows grouping, virtual columns cannot be used partialQuery.getAggregate() != null ? null : virtualColumnRegistry ) @@ -258,6 +272,25 @@ public static DruidQuery fromPartialQuery( sorting = null; } + if (partialQuery.getWindow() != null) { + final QueryContext queryContext = plannerContext.queryContext(); + if (queryContext.getBoolean(CTX_ENABLE_WINDOW_FNS, false)) { + windowing = Preconditions.checkNotNull( + Windowing.fromCalciteStuff( + partialQuery, + plannerContext, + sourceRowSignature, // Plans immediately after Scan, so safe to use the row signature from scan + rexBuilder + ) + ); + } else { + plannerContext.setPlanningError("Windowing Not Currently Supported"); + throw new CannotBuildQueryException("Windowing Not Currently Supported"); + } + } else { + windowing = null; + } + return new DruidQuery( dataSource, plannerContext, @@ -265,6 +298,7 @@ public static DruidQuery fromPartialQuery( selectProjection, grouping, sorting, + windowing, sourceRowSignature, outputRowType, virtualColumnRegistry @@ -404,9 +438,7 @@ private static Grouping computeGrouping( * @param plannerContext planner context * @param rowSignature source row signature * @param virtualColumnRegistry re-usable virtual column references - * * @return dimensions - * * @throws CannotBuildQueryException if dimensions cannot be computed */ private static List computeDimensions( @@ -438,7 +470,10 @@ private static List computeDimensions( final ColumnType outputType = Calcites.getColumnTypeForRelDataType(dataType); if (Types.isNullOr(outputType, ValueType.COMPLEX)) { // Can't group on unknown or COMPLEX types. - plannerContext.setPlanningError("SQL requires a group-by on a column of type %s that is unsupported.", outputType); + plannerContext.setPlanningError( + "SQL requires a group-by on a column of type %s that is unsupported.", + outputType + ); throw new CannotBuildQueryException(aggregate, rexNode); } @@ -511,9 +546,7 @@ private static Subtotals computeSubtotals( * @param finalizeAggregations true if this query should include explicit finalization for all of its * aggregators, where required. Useful for subqueries where Druid's native query layer * does not do this automatically. - * * @return aggregations - * * @throws CannotBuildQueryException if dimensions cannot be computed */ private static List computeAggregations( @@ -631,10 +664,13 @@ private static RowSignature computeOutputRowSignature( final RowSignature sourceRowSignature, @Nullable final Projection selectProjection, @Nullable final Grouping grouping, - @Nullable final Sorting sorting + @Nullable final Sorting sorting, + @Nullable final Windowing windowing ) { - if (sorting != null && sorting.getProjection() != null) { + if (windowing != null) { + return windowing.getSignature(); + } else if (sorting != null && sorting.getProjection() != null) { return sorting.getProjection().getOutputRowSignature(); } else if (grouping != null) { // Sanity check: cannot have both "grouping" and "selectProjection". @@ -791,7 +827,7 @@ private static Filtration toFiltration(DimFilter filter, VirtualColumnRegistry v /** * Whether the provided combination of dataSource, filtration, and queryGranularity is safe to use in queries. - * + *

* Necessary because some combinations are unsafe, mainly because they would lead to the creation of too many * time-granular buckets during query processing. */ @@ -870,6 +906,11 @@ public Query getQuery() */ private Query computeQuery() { + if (windowing != null) { + // Windowing can only be handled by window queries. + return toWindowQuery(); + } + if (dataSource instanceof QueryDataSource) { // If there is a subquery, then we prefer the outer query to be a groupBy if possible, since this potentially // enables more efficient execution. (The groupBy query toolchest can handle some subqueries by itself, without @@ -1312,6 +1353,26 @@ private GroupByQuery toGroupByQuery() return query.withOverriddenContext(theContext); } + /** + * Return this query as a {@link WindowOperatorQuery}, or null if this query cannot be run that way. + * + * @return query or null + */ + @Nullable + private WindowOperatorQuery toWindowQuery() + { + if (windowing == null) { + return null; + } + + return new WindowOperatorQuery( + dataSource, + plannerContext.queryContextMap(), + windowing.getSignature(), + windowing.getOperators() + ); + } + /** * Return this query as a Scan query, or null if this query is not compatible with Scan. * diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/PartialDruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/PartialDruidQuery.java index 15bcd5d4bdac..bdd4a4f735fa 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/PartialDruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/PartialDruidQuery.java @@ -28,6 +28,7 @@ import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; @@ -58,24 +59,28 @@ public class PartialDruidQuery private final Project aggregateProject; private final Sort sort; private final Project sortProject; + private final Window window; public enum Stage { // SCAN must be present on all queries. SCAN, - // WHERE_FILTER, SELECT_PROJECT may be present on any query. + // WHERE_FILTER, SELECT_PROJECT may be present on any query, except ones with WINDOW. WHERE_FILTER, SELECT_PROJECT, - // AGGREGATE, HAING_FILTER, AGGREGATE_PROJECT can only be present on aggregating queries. + // AGGREGATE, HAVING_FILTER, AGGREGATE_PROJECT can only be present on non-WINDOW aggregating queries. AGGREGATE, HAVING_FILTER, AGGREGATE_PROJECT, - // SORT, SORT_PROJECT may be present on any query. + // SORT, SORT_PROJECT may be present on any query, except ones with WINDOW. SORT, - SORT_PROJECT + SORT_PROJECT, + + // WINDOW may be present only together with SCAN. + WINDOW } private PartialDruidQuery( @@ -87,7 +92,8 @@ private PartialDruidQuery( final Project aggregateProject, final Filter havingFilter, final Sort sort, - final Project sortProject + final Project sortProject, + final Window window ) { this.builderSupplier = Preconditions.checkNotNull(builderSupplier, "builderSupplier"); @@ -99,6 +105,7 @@ private PartialDruidQuery( this.havingFilter = havingFilter; this.sort = sort; this.sortProject = sortProject; + this.window = window; } public static PartialDruidQuery create(final RelNode scanRel) @@ -107,7 +114,7 @@ public static PartialDruidQuery create(final RelNode scanRel) scanRel.getCluster(), scanRel.getTable() != null ? scanRel.getTable().getRelOptSchema() : null ); - return new PartialDruidQuery(builderSupplier, scanRel, null, null, null, null, null, null, null); + return new PartialDruidQuery(builderSupplier, scanRel, null, null, null, null, null, null, null, null); } public RelNode getScan() @@ -150,6 +157,11 @@ public Project getSortProject() return sortProject; } + public Window getWindow() + { + return window; + } + public PartialDruidQuery withWhereFilter(final Filter newWhereFilter) { validateStage(Stage.WHERE_FILTER); @@ -162,7 +174,8 @@ public PartialDruidQuery withWhereFilter(final Filter newWhereFilter) aggregateProject, havingFilter, sort, - sortProject + sortProject, + window ); } @@ -204,7 +217,8 @@ public PartialDruidQuery withSelectProject(final Project newSelectProject) aggregateProject, havingFilter, sort, - sortProject + sortProject, + window ); } @@ -220,7 +234,8 @@ public PartialDruidQuery withAggregate(final Aggregate newAggregate) aggregateProject, havingFilter, sort, - sortProject + sortProject, + window ); } @@ -236,7 +251,8 @@ public PartialDruidQuery withHavingFilter(final Filter newHavingFilter) aggregateProject, newHavingFilter, sort, - sortProject + sortProject, + window ); } @@ -252,7 +268,8 @@ public PartialDruidQuery withAggregateProject(final Project newAggregateProject) newAggregateProject, havingFilter, sort, - sortProject + sortProject, + window ); } @@ -268,7 +285,8 @@ public PartialDruidQuery withSort(final Sort newSort) aggregateProject, havingFilter, newSort, - sortProject + sortProject, + window ); } @@ -284,7 +302,25 @@ public PartialDruidQuery withSortProject(final Project newSortProject) aggregateProject, havingFilter, sort, - newSortProject + newSortProject, + window + ); + } + + public PartialDruidQuery withWindow(final Window newWindow) + { + validateStage(Stage.WINDOW); + return new PartialDruidQuery( + builderSupplier, + scan, + whereFilter, + selectProject, + aggregate, + aggregateProject, + havingFilter, + sort, + sortProject, + newWindow ); } @@ -341,7 +377,10 @@ public boolean canAccept(final Stage stage) { final Stage currentStage = stage(); - if (currentStage == Stage.SELECT_PROJECT && stage == Stage.SELECT_PROJECT) { + if (stage == Stage.WINDOW) { + // Special case: WINDOW can only be provided along with SCAN. + return currentStage == Stage.SCAN; + } else if (currentStage == Stage.SELECT_PROJECT && stage == Stage.SELECT_PROJECT) { // Special case: allow layering SELECT_PROJECT on top of SELECT_PROJECT. Calcite's builtin rules cannot // always collapse these, so we have to (one example: testSemiJoinWithOuterTimeExtract). See // withSelectProject for the code here that handles this. @@ -352,12 +391,9 @@ public boolean canAccept(final Stage stage) } else if (stage.compareTo(Stage.AGGREGATE) > 0 && stage.compareTo(Stage.SORT) < 0 && aggregate == null) { // Cannot do post-aggregation stages without an aggregation. return false; - } else if (stage.compareTo(Stage.SORT) > 0 && sort == null) { - // Cannot do post-sort stages without a sort. - return false; } else { - // Looks good. - return true; + // If we are after the SORT phase, make sure we have a sort... + return stage.compareTo(Stage.SORT) <= 0 || sort != null; } } @@ -370,7 +406,9 @@ public boolean canAccept(final Stage stage) @SuppressWarnings("VariableNotUsedInsideIf") public Stage stage() { - if (sortProject != null) { + if (window != null) { + return Stage.WINDOW; + } else if (sortProject != null) { return Stage.SORT_PROJECT; } else if (sort != null) { return Stage.SORT; @@ -399,6 +437,8 @@ public RelNode leafRel() final Stage currentStage = stage(); switch (currentStage) { + case WINDOW: + return window; case SORT_PROJECT: return sortProject; case SORT: diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java new file mode 100644 index 000000000000..5cbffc6d2a5b --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.rel; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Window; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.operator.NaivePartitioningOperatorFactory; +import org.apache.druid.query.operator.OperatorFactory; +import org.apache.druid.query.operator.WindowOperatorFactory; +import org.apache.druid.query.operator.window.ComposingProcessor; +import org.apache.druid.query.operator.window.Processor; +import org.apache.druid.query.operator.window.WindowAggregateProcessor; +import org.apache.druid.query.operator.window.ranking.WindowCumeDistProcessor; +import org.apache.druid.query.operator.window.ranking.WindowDenseRankProcessor; +import org.apache.druid.query.operator.window.ranking.WindowPercentileProcessor; +import org.apache.druid.query.operator.window.ranking.WindowRankProcessor; +import org.apache.druid.query.operator.window.ranking.WindowRowNumberProcessor; +import org.apache.druid.query.operator.window.value.WindowFirstProcessor; +import org.apache.druid.query.operator.window.value.WindowLastProcessor; +import org.apache.druid.query.operator.window.value.WindowOffsetProcessor; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.sql.calcite.aggregation.Aggregation; +import org.apache.druid.sql.calcite.expression.DruidExpression; +import org.apache.druid.sql.calcite.expression.Expressions; +import org.apache.druid.sql.calcite.planner.Calcites; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.rule.GroupByRules; +import org.apache.druid.sql.calcite.table.RowSignatures; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Maps onto a {@link org.apache.druid.query.operator.WindowOperatorQuery}. + */ +public class Windowing +{ + private static final ImmutableMap KNOWN_WINDOW_FNS = ImmutableMap + .builder() + .put("LAG", (agg) -> new WindowOffsetProcessor(agg.getColumn(0), agg.getOutputName(), -agg.getConstantInt(1))) + .put("LEAD", (agg) -> new WindowOffsetProcessor(agg.getColumn(0), agg.getOutputName(), agg.getConstantInt(1))) + .put("FIRST_VALUE", (agg) -> new WindowFirstProcessor(agg.getColumn(0), agg.getOutputName())) + .put("LAST_VALUE", (agg) -> new WindowLastProcessor(agg.getColumn(0), agg.getOutputName())) + .put("CUME_DIST", (agg) -> new WindowCumeDistProcessor(agg.getOrderingColumns(), agg.getOutputName())) + .put("DENSE_RANK", (agg) -> new WindowDenseRankProcessor(agg.getOrderingColumns(), agg.getOutputName())) + .put("NTILE", (agg) -> new WindowPercentileProcessor(agg.getOutputName(), agg.getConstantInt(0))) + .put("PERCENT_RANK", (agg) -> new WindowRankProcessor(agg.getOrderingColumns(), agg.getOutputName(), true)) + .put("RANK", (agg) -> new WindowRankProcessor(agg.getOrderingColumns(), agg.getOutputName(), false)) + .put("ROW_NUMBER", (agg) -> new WindowRowNumberProcessor(agg.getOutputName())) + .build(); + private final List ops; + + @Nonnull + public static Windowing fromCalciteStuff( + final PartialDruidQuery partialQuery, + final PlannerContext plannerContext, + final RowSignature rowSignature, + final RexBuilder rexBuilder + ) + { + final Window window = Preconditions.checkNotNull(partialQuery.getWindow(), "window"); + + // TODO(gianm): insert sorts and split the groups up at the rule stage; by this time, we assume there's one + // window and the dataset is already sorted appropriately. + if (window.groups.size() != 1) { + plannerContext.setPlanningError("Multiple windows are not supported"); + throw new CannotBuildQueryException(window); + } + final Window.Group group = Iterables.getOnlyElement(window.groups); + + // Window. + // TODO(gianm): Validate order-by keys instead of ignoring them. + + final List partitionColumns = new ArrayList<>(); + for (int groupKey : group.keys) { + partitionColumns.add(rowSignature.getColumnName(groupKey)); + } + + // Frame. + // TODO(gianm): Validate ROWS vs RANGE instead of ignoring it. + // TODO(gianm): Support various other kinds of frames. + if (!group.lowerBound.isUnbounded()) { + plannerContext.setPlanningError("Lower bound [%s] is not supported", group.upperBound); + throw new CannotBuildQueryException(window); + } + + final boolean cumulative; + if (group.upperBound.isUnbounded()) { + cumulative = false; + } else if (group.upperBound.isCurrentRow()) { + cumulative = true; + } else { + plannerContext.setPlanningError("Upper bound [%s] is not supported", group.upperBound); + throw new CannotBuildQueryException(window); + } + + // Aggregations. + final String outputNamePrefix = Calcites.findUnusedPrefixForDigits("w", rowSignature.getColumnNames()); + final List aggregateCalls = group.getAggregateCalls(window); + + final List processors = new ArrayList<>(); + final List aggregations = new ArrayList<>(); + final List expectedOutputColumns = new ArrayList<>(rowSignature.getColumnNames()); + + for (int i = 0; i < aggregateCalls.size(); i++) { + final String aggName = outputNamePrefix + i; + expectedOutputColumns.add(aggName); + + final AggregateCall aggCall = aggregateCalls.get(i); + + ProcessorMaker maker = KNOWN_WINDOW_FNS.get(aggCall.getAggregation().getName()); + if (maker == null) { + final Aggregation aggregation = GroupByRules.translateAggregateCall( + plannerContext, + rowSignature, + null, + rexBuilder, + partialQuery.getSelectProject(), + Collections.emptyList(), + aggName, + aggCall, + false // TODO: finalize in a separate operator + ); + + if (aggregation == null + || aggregation.getPostAggregator() != null + || aggregation.getAggregatorFactories().size() != 1) { + if (null == plannerContext.getPlanningError()) { + plannerContext.setPlanningError("Aggregation [%s] is not supported", aggCall); + } + throw new CannotBuildQueryException(window, aggCall); + } + + aggregations.add(Iterables.getOnlyElement(aggregation.getAggregatorFactories())); + } else { + processors.add(maker.make( + new WindowAggregate( + aggName, + aggCall, + rowSignature, + plannerContext, + partialQuery.getSelectProject(), + window.constants, + group + ) + )); + } + } + + if (!aggregations.isEmpty()) { + if (cumulative) { + processors.add(new WindowAggregateProcessor(null, aggregations)); + } else { + processors.add(new WindowAggregateProcessor(aggregations, null)); + } + } + + if (processors.isEmpty()) { + throw new ISE("No processors from Window[%s], why was this code called?", window); + } + + final List ops = Arrays.asList( + new NaivePartitioningOperatorFactory(partitionColumns), + new WindowOperatorFactory( + processors.size() == 1 ? + processors.get(0) : new ComposingProcessor(processors.toArray(new Processor[0])) + ) + ); + + return new Windowing( + RowSignatures.fromRelDataType(expectedOutputColumns, window.getRowType()), + ops + ); + } + + private final RowSignature signature; + + public Windowing( + final RowSignature signature, + List ops + ) + { + this.signature = signature; + this.ops = ops; + } + + public RowSignature getSignature() + { + return signature; + } + + public List getOperators() + { + return ops; + } + + private interface ProcessorMaker + { + Processor make(WindowAggregate agg); + } + + private static class WindowAggregate + { + private final String outputName; + private final AggregateCall call; + private final RowSignature sig; + private final PlannerContext context; + private final Project project; + private final List constants; + private final Window.Group group; + + private WindowAggregate( + String outputName, + AggregateCall call, + RowSignature sig, + PlannerContext context, + Project project, + List constants, + Window.Group group + ) + { + this.outputName = outputName; + this.call = call; + this.sig = sig; + this.context = context; + this.project = project; + this.constants = constants; + this.group = group; + + if (project != null) { + throw new ISE("Suddenly, the project[%s] is no longer null, the code might need to change.", project); + } + } + + public String getOutputName() + { + return outputName; + } + + public ArrayList getOrderingColumns() + { + final List fields = group.orderKeys.getFieldCollations(); + ArrayList retVal = new ArrayList<>(fields.size()); + for (RelFieldCollation field : fields) { + retVal.add(sig.getColumnName(field.getFieldIndex())); + } + return retVal; + } + + public String getColumn(int argPosition) + { + RexNode columnArgument = Expressions.fromFieldAccess(sig, project, call.getArgList().get(argPosition)); + final DruidExpression expression = Expressions.toDruidExpression(context, sig, columnArgument); + return expression.getDirectColumn(); + } + + public RexLiteral getConstantArgument(int argPosition) + { + final Integer constantIndex = call.getArgList().get(argPosition) - sig.size(); + return constants.get(constantIndex); + } + + public int getConstantInt(int argPosition) + { + return ((Number) getConstantArgument(argPosition).getValue()).intValue(); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java index 113f37d896fb..8316b1e9868d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java @@ -28,12 +28,15 @@ import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.core.Window; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.rel.DruidOuterQueryRel; +import org.apache.druid.sql.calcite.rel.DruidQuery; import org.apache.druid.sql.calcite.rel.DruidRel; import org.apache.druid.sql.calcite.rel.PartialDruidQuery; +import java.util.ArrayList; import java.util.List; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -50,51 +53,59 @@ private DruidRules() public static List rules(PlannerContext plannerContext) { - return ImmutableList.of( - new DruidQueryRule<>( - Filter.class, - PartialDruidQuery.Stage.WHERE_FILTER, - PartialDruidQuery::withWhereFilter - ), - new DruidQueryRule<>( - Project.class, - PartialDruidQuery.Stage.SELECT_PROJECT, - PartialDruidQuery::withSelectProject - ), - new DruidQueryRule<>( - Aggregate.class, - PartialDruidQuery.Stage.AGGREGATE, - PartialDruidQuery::withAggregate - ), - new DruidQueryRule<>( - Project.class, - PartialDruidQuery.Stage.AGGREGATE_PROJECT, - PartialDruidQuery::withAggregateProject - ), - new DruidQueryRule<>( - Filter.class, - PartialDruidQuery.Stage.HAVING_FILTER, - PartialDruidQuery::withHavingFilter - ), - new DruidQueryRule<>( - Sort.class, - PartialDruidQuery.Stage.SORT, - PartialDruidQuery::withSort - ), - new DruidQueryRule<>( - Project.class, - PartialDruidQuery.Stage.SORT_PROJECT, - PartialDruidQuery::withSortProject - ), - DruidOuterQueryRule.AGGREGATE, - DruidOuterQueryRule.WHERE_FILTER, - DruidOuterQueryRule.SELECT_PROJECT, - DruidOuterQueryRule.SORT, - new DruidUnionRule(plannerContext), - new DruidUnionDataSourceRule(plannerContext), - DruidSortUnionRule.instance(), - DruidJoinRule.instance(plannerContext) + final ArrayList retVal = new ArrayList<>( + ImmutableList.of( + new DruidQueryRule<>( + Filter.class, + PartialDruidQuery.Stage.WHERE_FILTER, + PartialDruidQuery::withWhereFilter + ), + new DruidQueryRule<>( + Project.class, + PartialDruidQuery.Stage.SELECT_PROJECT, + PartialDruidQuery::withSelectProject + ), + new DruidQueryRule<>( + Aggregate.class, + PartialDruidQuery.Stage.AGGREGATE, + PartialDruidQuery::withAggregate + ), + new DruidQueryRule<>( + Project.class, + PartialDruidQuery.Stage.AGGREGATE_PROJECT, + PartialDruidQuery::withAggregateProject + ), + new DruidQueryRule<>( + Filter.class, + PartialDruidQuery.Stage.HAVING_FILTER, + PartialDruidQuery::withHavingFilter + ), + new DruidQueryRule<>( + Sort.class, + PartialDruidQuery.Stage.SORT, + PartialDruidQuery::withSort + ), + new DruidQueryRule<>( + Project.class, + PartialDruidQuery.Stage.SORT_PROJECT, + PartialDruidQuery::withSortProject + ), + DruidOuterQueryRule.AGGREGATE, + DruidOuterQueryRule.WHERE_FILTER, + DruidOuterQueryRule.SELECT_PROJECT, + DruidOuterQueryRule.SORT, + new DruidUnionRule(plannerContext), + new DruidUnionDataSourceRule(plannerContext), + DruidSortUnionRule.instance(), + DruidJoinRule.instance(plannerContext) + ) ); + + if (plannerContext.queryContext().getBoolean(DruidQuery.CTX_ENABLE_WINDOW_FNS, false)) { + retVal.add(new DruidQueryRule<>(Window.class, PartialDruidQuery.Stage.WINDOW, PartialDruidQuery::withWindow)); + retVal.add(DruidOuterQueryRule.WINDOW); + } + return retVal; } public static class DruidQueryRule extends RelOptRule @@ -228,6 +239,28 @@ public void onMatch(final RelOptRuleCall call) } }; + public static final RelOptRule WINDOW = new DruidOuterQueryRule( + operand(Window.class, operandJ(DruidRel.class, null, CAN_BUILD_ON, any())), + "WINDOW" + ) + { + @Override + public void onMatch(final RelOptRuleCall call) + { + final Window window = call.rel(0); + final DruidRel druidRel = call.rel(1); + + final DruidOuterQueryRel outerQueryRel = DruidOuterQueryRel.create( + druidRel, + PartialDruidQuery.create(druidRel.getPartialDruidQuery().leafRel()) + .withWindow(window) + ); + if (outerQueryRel.isValidDruidQuery()) { + call.transformTo(outerQueryRel); + } + } + }; + public DruidOuterQueryRule(final RelOptRuleOperand op, final String description) { super(op, StringUtils.format("%s(%s)", DruidOuterQueryRel.class.getSimpleName(), description)); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/GroupByRules.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/GroupByRules.java index cf596ce7f546..549c7899730a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/GroupByRules.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/GroupByRules.java @@ -34,6 +34,7 @@ import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -54,7 +55,7 @@ private GroupByRules() public static Aggregation translateAggregateCall( final PlannerContext plannerContext, final RowSignature rowSignature, - final VirtualColumnRegistry virtualColumnRegistry, + @Nullable final VirtualColumnRegistry virtualColumnRegistry, final RexBuilder rexBuilder, final Project project, final List existingAggregations, diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java index a9fba5ee3580..258a744827a4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java @@ -240,9 +240,8 @@ private QueryResponse mapResultSequence( mapping[i] = idx; } - //noinspection unchecked final Sequence sequence = toolChest.resultsAsArrays(query, results.getResults()); - return new QueryResponse( + return new QueryResponse<>( Sequences.map( sequence, array -> { diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 9db98505dc1f..748ed20ce3c5 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -559,6 +559,12 @@ public void testDatabaseMetaDataTables() throws SQLException Pair.of("TABLE_NAME", CalciteTests.USERVISITDATASOURCE), Pair.of("TABLE_SCHEM", "druid"), Pair.of("TABLE_TYPE", "TABLE") + ), + row( + Pair.of("TABLE_CAT", "druid"), + Pair.of("TABLE_NAME", "wikipedia"), + Pair.of("TABLE_SCHEM", "druid"), + Pair.of("TABLE_TYPE", "TABLE") ) ), getRows( @@ -633,6 +639,12 @@ public void testDatabaseMetaDataTablesAsSuperuser() throws SQLException Pair.of("TABLE_NAME", CalciteTests.USERVISITDATASOURCE), Pair.of("TABLE_SCHEM", "druid"), Pair.of("TABLE_TYPE", "TABLE") + ), + row( + Pair.of("TABLE_CAT", "druid"), + Pair.of("TABLE_NAME", "wikipedia"), + Pair.of("TABLE_SCHEM", "druid"), + Pair.of("TABLE_TYPE", "TABLE") ) ), getRows( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index fada41896009..7254e84c51f8 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -114,7 +114,6 @@ import org.junit.rules.TemporaryFolder; import javax.annotation.Nullable; - import java.io.IOException; import java.util.Arrays; import java.util.HashMap; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 516e363f9bb3..c9df143d9ca8 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -218,6 +218,7 @@ public void testInformationSchemaTables() .add(new Object[]{"druid", CalciteTests.SOME_DATASOURCE, "TABLE", "NO", "NO"}) .add(new Object[]{"druid", CalciteTests.SOMEXDATASOURCE, "TABLE", "NO", "NO"}) .add(new Object[]{"druid", CalciteTests.USERVISITDATASOURCE, "TABLE", "NO", "NO"}) + .add(new Object[]{"druid", "wikipedia", "TABLE", "NO", "NO"}) .add(new Object[]{"INFORMATION_SCHEMA", "COLUMNS", "SYSTEM_TABLE", "NO", "NO"}) .add(new Object[]{"INFORMATION_SCHEMA", "SCHEMATA", "SYSTEM_TABLE", "NO", "NO"}) .add(new Object[]{"INFORMATION_SCHEMA", "TABLES", "SYSTEM_TABLE", "NO", "NO"}) @@ -254,6 +255,7 @@ public void testInformationSchemaTables() .add(new Object[]{"druid", CalciteTests.SOME_DATASOURCE, "TABLE", "NO", "NO"}) .add(new Object[]{"druid", CalciteTests.SOMEXDATASOURCE, "TABLE", "NO", "NO"}) .add(new Object[]{"druid", CalciteTests.USERVISITDATASOURCE, "TABLE", "NO", "NO"}) + .add(new Object[]{"druid", "wikipedia", "TABLE", "NO", "NO"}) .add(new Object[]{"INFORMATION_SCHEMA", "COLUMNS", "SYSTEM_TABLE", "NO", "NO"}) .add(new Object[]{"INFORMATION_SCHEMA", "SCHEMATA", "SYSTEM_TABLE", "NO", "NO"}) .add(new Object[]{"INFORMATION_SCHEMA", "TABLES", "SYSTEM_TABLE", "NO", "NO"}) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java new file mode 100644 index 000000000000..0a12eb8c88ea --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.common.collect.ImmutableMap; +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import junitparams.naming.TestCaseName; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.query.Query; +import org.apache.druid.query.operator.OperatorFactory; +import org.apache.druid.query.operator.WindowOperatorQuery; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.regex.Pattern; + +@RunWith(JUnitParamsRunner.class) +public class CalciteWindowQueryTest extends BaseCalciteQueryTest +{ + + public static final boolean DUMP_EXPECTED_RESULTS = Boolean.parseBoolean( + System.getProperty("druid.tests.sql.dumpExpectedResults") + ); + + static { + NullHandling.initializeForTests(); + } + + private static final ObjectMapper YAML_JACKSON = new DefaultObjectMapper(new YAMLFactory(), "tests"); + + private static final AtomicLong EXPECTED_TESTS = new AtomicLong(); + private static final AtomicLong TEST_COUNTER = new AtomicLong(); + + public Object parametersForWindowQueryTest() throws Exception + { + final URL windowFolderUrl = ClassLoader.getSystemResource("calcite/tests/window"); + File windowFolder = new File(windowFolderUrl.toURI()); + + final File[] listedFiles = windowFolder.listFiles( + pathname -> pathname.getName().toLowerCase(Locale.ROOT).endsWith(".sqltest") + ); + EXPECTED_TESTS.set(listedFiles.length); + + Pattern matcher = Pattern.compile(".*"); + + return Arrays + .stream(Objects.requireNonNull(listedFiles)) + .map(File::getName) + .filter(matcher.asPredicate()) + .toArray(); + } + + @AfterClass + public static void testRanAllTests() + { + // This validation exists to catch issues with the filter Pattern accidentally getting checked in. It validates + // that we ran all of the tests from the directory. If this is failing, most likely, the filter Pattern in + // parametersForWindowQueryTest accidentally got checked in as something other than ".*" + Assert.assertEquals(EXPECTED_TESTS.get(), TEST_COUNTER.get()); + } + + @Test + @Parameters(method = "parametersForWindowQueryTest") + @SuppressWarnings("unchecked") + @TestCaseName("{0}") + public void windowQueryTest(String filename) throws IOException + { + final Function stringManipulator; + if (NullHandling.sqlCompatible()) { + stringManipulator = s -> "".equals(s) ? null : s; + } else { + stringManipulator = Function.identity(); + } + + TEST_COUNTER.incrementAndGet(); + final URL systemResource = ClassLoader.getSystemResource("calcite/tests/window/" + filename); + + final Object objectFromYaml = YAML_JACKSON.readValue(systemResource.openStream(), Object.class); + + final ObjectMapper queryJackson = queryFramework().queryJsonMapper(); + final WindowQueryTestInputClass input = queryJackson.convertValue(objectFromYaml, WindowQueryTestInputClass.class); + + Function jacksonToString = value -> { + try { + return queryJackson.writeValueAsString(value); + } + catch (JsonProcessingException e) { + throw new RE(e); + } + }; + + if ("operatorValidation".equals(input.type)) { + testBuilder() + .skipVectorize(true) + .sql(input.sql) + .queryContext(ImmutableMap.of("windowsAreForClosers", true)) + .addCustomVerification(QueryVerification.ofResults(results -> { + if (results.exception != null) { + throw new RE(results.exception, "Failed to execute because of exception."); + } + + Assert.assertEquals(1, results.recordedQueries.size()); + + final WindowOperatorQuery query = (WindowOperatorQuery) results.recordedQueries.get(0); + for (int i = 0; i < input.expectedOperators.size(); ++i) { + final OperatorFactory expectedOperator = input.expectedOperators.get(i); + final OperatorFactory actualOperator = query.getOperators().get(i); + if (!expectedOperator.validateEquivalent(actualOperator)) { + // This assertion always fails because the validate equivalent failed, but we do it anyway + // so that we get values in the output of the failed test to make it easier to + // debug what happened. Note, we use the Jackson representation when showing the diff. There is + // a chance that this representation is exactly equivalent, but the validation call is still failing + // this is probably indicative of a bug where something that needs to be serialized by Jackson + // currently is not. Check your getters. + + // prepend different values so that we are guaranteed that it is always different + String expected = "e " + jacksonToString.apply(expectedOperator); + String actual = "a " + jacksonToString.apply(actualOperator); + + Assert.assertEquals("Operator Mismatch, index[" + i + "]", expected, actual); + } + } + final RowSignature outputSignature = query.getRowSignature(); + ColumnType[] types = new ColumnType[outputSignature.size()]; + for (int i = 0; i < outputSignature.size(); ++i) { + types[i] = outputSignature.getColumnType(i).get(); + Assert.assertEquals(types[i], results.signature.getColumnType(i).get()); + } + + maybeDumpExpectedResults(jacksonToString, results.results); + for (Object[] result : input.expectedResults) { + for (int i = 0; i < types.length; i++) { + // Jackson deserializes numbers as the minimum size required to store the value. This means that + // Longs can become Integer objects and then they fail equality checks. We read the expected + // results using Jackson, so, we coerce the expected results to the type expected. + if (result[i] != null) { + if (result[i] instanceof Number) { + switch (types[i].getType()) { + case LONG: + result[i] = ((Number) result[i]).longValue(); + break; + case DOUBLE: + result[i] = ((Number) result[i]).doubleValue(); + break; + case FLOAT: + result[i] = ((Number) result[i]).floatValue(); + break; + default: + throw new ISE("result[%s] was type[%s]!? Expected it to be numerical", i, types[i].getType()); + } + } else if (result[i] instanceof String) { + result[i] = stringManipulator.apply((String) result[i]); + } + } + } + } + assertResultsEquals(filename, input.expectedResults, results.results); + })) + .run(); + } + } + + private void maybeDumpExpectedResults( + Function toStrFn, List results + ) + { + if (DUMP_EXPECTED_RESULTS) { + for (Object[] result : results) { + System.out.println(" - " + toStrFn.apply(result)); + } + } + } + + public static class WindowQueryTestInputClass + { + @JsonProperty + public String type; + + @JsonProperty + public String sql; + + @JsonProperty + public Query nativeQuery; + + @JsonProperty + public List expectedOperators; + + @JsonProperty + public List expectedResults; + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestBuilder.java b/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestBuilder.java index f9b843d5d8b3..e0a9320002dd 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestBuilder.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestBuilder.java @@ -37,7 +37,7 @@ import org.junit.rules.ExpectedException; import javax.annotation.Nullable; - +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -69,8 +69,11 @@ public class QueryTestBuilder public interface QueryTestConfig { QueryLogHook queryLogHook(); + ExpectedException expectedException(); + ObjectMapper jsonMapper(); + PlannerFixture plannerFixture(PlannerConfig plannerConfig, AuthConfig authConfig); ResultsVerifier defaultResultsVerifier(List expectedResults, RowSignature expectedResultSignature); } @@ -83,10 +86,12 @@ public interface QueryTestConfig protected AuthenticationResult authenticationResult = CalciteTests.REGULAR_USER_AUTH_RESULT; protected List> expectedQueries; protected List expectedResults; + protected List customVerifications = new ArrayList<>(); protected RowSignature expectedResultSignature; protected List expectedResources; protected ResultsVerifier expectedResultsVerifier; - protected @Nullable Consumer expectedExceptionInitializer; + @Nullable + protected Consumer expectedExceptionInitializer; protected boolean skipVectorize; protected boolean queryCannotVectorize; protected AuthConfig authConfig = new AuthConfig(); @@ -151,6 +156,23 @@ public QueryTestBuilder expectedResults( return this; } + public QueryTestBuilder addCustomVerification( + QueryTestRunner.QueryVerifyStepFactory factory + ) + { + this.customVerifications.add(factory); + return this; + } + + public QueryTestBuilder setCustomVerifications( + List factories + ) + { + this.customVerifications = new ArrayList<>(); + this.customVerifications.addAll(factories); + return this; + } + public QueryTestBuilder expectedSignature( final RowSignature expectedResultSignature ) @@ -259,4 +281,5 @@ public QueryResults results() { return build().resultsOnly(); } + } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestRunner.java b/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestRunner.java index f3e1661b09ac..8b64f7a429bc 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestRunner.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestRunner.java @@ -69,6 +69,11 @@ */ public class QueryTestRunner { + public interface QueryVerifyStepFactory + { + QueryVerifyStep make(ExecuteQuery execStep); + } + /** * Test step that executes or prepares a query. */ @@ -291,26 +296,15 @@ public static Pair> getResults( } /** - * Base class for steps which validate query execution results. + * Verify query results. */ - public abstract static class VerifyExecStep implements QueryVerifyStep + public static class VerifyResults implements QueryVerifyStep { protected final ExecuteQuery execStep; - public VerifyExecStep(ExecuteQuery execStep) - { - this.execStep = execStep; - } - } - - /** - * Verify query results. - */ - public static class VerifyResults extends VerifyExecStep - { public VerifyResults(ExecuteQuery execStep) { - super(execStep); + this.execStep = execStep; } @Override @@ -341,11 +335,13 @@ private void verifyResults(QueryResults queryResults) * Verify the native queries generated by an execution run against a set * provided in the builder. */ - public static class VerifyNativeQueries extends VerifyExecStep + public static class VerifyNativeQueries implements QueryVerifyStep { + protected final ExecuteQuery execStep; + public VerifyNativeQueries(ExecuteQuery execStep) { - super(execStep); + this.execStep = execStep; } @Override @@ -364,6 +360,17 @@ private void verifyQuery(QueryResults queryResults) QueryTestBuilder builder = execStep.builder(); final List> expectedQueries = new ArrayList<>(); for (Query query : builder.expectedQueries) { + // The tests set a lot of various values in the context that are not relevant to how the query actually planned, + // so we effectively ignore these keys in the context during query validation by overwriting whatever + // context had been set in the test with the context produced by the test setup code. This means that any + // context parameter that the tests choose to set will never actually be tested (it will always be overridden) + // while parameters that don't get set by the test can be tested. + // + // This is pretty magical, it would probably be a good thing to move away from this hard-to-predict setting + // of context parameters towards a test setup that is much more explicit and easier to understand. Perhaps + // we could have validations of query objects that are a bit more intelligent. That is, instead of relying on + // equals, perhaps we could have a context validator that only validates that keys set on the expected query + // are set, allowing any other context keys to also be set? expectedQueries.add(BaseCalciteQueryTest.recursivelyOverrideContext(query, queryResults.queryContext)); } @@ -447,11 +454,13 @@ public void verify() /** * Verify resources for a prepared query against the expected list. */ - public static class VerifyExecuteSignature extends VerifyExecStep + public static class VerifyExecuteSignature implements QueryVerifyStep { + private final ExecuteQuery execStep; + public VerifyExecuteSignature(ExecuteQuery execStep) { - super(execStep); + this.execStep = execStep; } @Override @@ -467,11 +476,13 @@ public void verify() } } - public static class VerifyLogicalPlan extends VerifyExecStep + public static class VerifyLogicalPlan implements QueryVerifyStep { + private final ExecuteQuery execStep; + public VerifyLogicalPlan(ExecuteQuery execStep) { - super(execStep); + this.execStep = execStep; } @Override @@ -531,11 +542,13 @@ private String visualizePlan(PlannerCaptureHook hook) * after the first failure. It would be better to check all three * runs, but that's an exercise for later. */ - public static class VerifyExpectedException extends VerifyExecStep + public static class VerifyExpectedException implements QueryVerifyStep { + protected final ExecuteQuery execStep; + public VerifyExpectedException(ExecuteQuery execStep) { - super(execStep); + this.execStep = execStep; } @Override @@ -622,6 +635,12 @@ public QueryTestRunner(QueryTestBuilder builder) verifySteps.add(new QueryTestRunner.VerifyResults(execStep)); } + if (!builder.customVerifications.isEmpty()) { + for (QueryTestRunner.QueryVerifyStepFactory customVerification : builder.customVerifications) { + verifySteps.add(customVerification.make(execStep)); + } + } + // The exception is always verified: either there should be no exception // (the other steps ran), or there should be the defined exception. verifySteps.add(new QueryTestRunner.VerifyExpectedException(execStep)); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/QueryVerification.java b/sql/src/test/java/org/apache/druid/sql/calcite/QueryVerification.java new file mode 100644 index 000000000000..ba57e603b372 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/QueryVerification.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +public class QueryVerification +{ + public static QueryResultsVerifierFactory ofResults(QueryResultsVerifier verifier) + { + return new QueryResultsVerifierFactory(verifier); + } + + public interface QueryResultsVerifier + { + void verifyResults(QueryTestRunner.QueryResults results); + } + + public static class QueryResultsVerifierFactory implements QueryTestRunner.QueryVerifyStepFactory + { + private final QueryResultsVerifier verifier; + + public QueryResultsVerifierFactory( + QueryResultsVerifier verifier + ) + { + this.verifier = verifier; + } + + @Override + public QueryTestRunner.QueryVerifyStep make(QueryTestRunner.ExecuteQuery execStep) + { + return () -> { + for (QueryTestRunner.QueryResults queryResults : execStep.results()) { + verifier.verifyResults(queryResults); + } + }; + } + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java index ca312983ffd6..5ca07c83afc1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java @@ -103,7 +103,6 @@ public SpecificSegmentsQuerySegmentWalker( this.walker = QueryStackTests.createClientQuerySegmentWalker( QueryStackTests.createClusterQuerySegmentWalker( timelines, - joinableFactoryWrapper, conglomerate, scheduler ), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index cace9647e96a..051ad3477327 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -62,7 +62,6 @@ import org.apache.druid.timeline.DataSegment; import javax.inject.Singleton; - import java.io.File; import java.io.IOException; import java.util.Properties; @@ -473,13 +472,15 @@ public void configure(Binder binder) binder.bind(DataSegment.PruneSpecsHolder.class).toInstance(DataSegment.PruneSpecsHolder.DEFAULT); } - @Provides @Singleton + @Provides + @Singleton public QueryRunnerFactoryConglomerate conglomerate() { return componentSupplier.createCongolmerate(builder, resourceCloser); } - @Provides @Singleton + @Provides + @Singleton public JoinableFactoryWrapper joinableFactoryWrapper(final Injector injector) { return builder.componentSupplier.createJoinableFactoryWrapper( @@ -487,7 +488,8 @@ public JoinableFactoryWrapper joinableFactoryWrapper(final Injector injector) ); } - @Provides @Singleton + @Provides + @LazySingleton public SpecificSegmentsQuerySegmentWalker segmentsQuerySegmentWalker(final Injector injector) { try { @@ -504,7 +506,8 @@ public SpecificSegmentsQuerySegmentWalker segmentsQuerySegmentWalker(final Injec } } - @Provides @Singleton + @Provides + @Singleton public QueryLifecycleFactory queryLifecycleFactory(final Injector injector) { return QueryFrameworkUtils.createMockQueryLifecycleFactory( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java index d8daa6595594..07a9fbc30abf 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java @@ -19,6 +19,8 @@ package org.apache.druid.sql.calcite.util; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -35,7 +37,10 @@ import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.RE; import org.apache.druid.query.DataSource; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.InlineDataSource; @@ -62,16 +67,24 @@ import org.apache.druid.server.QueryStackTests; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpec; import org.joda.time.DateTime; import org.joda.time.chrono.ISOChronology; +import java.io.BufferedReader; import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; /** * Builds a set of test data used by the Calcite query tests. The test data is @@ -119,14 +132,21 @@ public Optional build( new TimestampSpec(TIMESTAMP_COLUMN, "iso", null), new DimensionsSpec( ImmutableList.builder() - .addAll(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim2", "dim3", "dim4", "dim5", "dim6"))) - .add(new DoubleDimensionSchema("d1")) - .add(new DoubleDimensionSchema("d2")) - .add(new FloatDimensionSchema("f1")) - .add(new FloatDimensionSchema("f2")) - .add(new LongDimensionSchema("l1")) - .add(new LongDimensionSchema("l2")) - .build() + .addAll(DimensionsSpec.getDefaultSchemas(ImmutableList.of( + "dim1", + "dim2", + "dim3", + "dim4", + "dim5", + "dim6" + ))) + .add(new DoubleDimensionSchema("d1")) + .add(new DoubleDimensionSchema("d2")) + .add(new FloatDimensionSchema("f1")) + .add(new FloatDimensionSchema("f2")) + .add(new LongDimensionSchema("l1")) + .add(new LongDimensionSchema("l2")) + .build() ) ) ); @@ -137,18 +157,18 @@ public Optional build( new DimensionsSpec( DimensionsSpec.getDefaultSchemas( ImmutableList.builder().add("dimHyperUnique") - .add("dimMultivalEnumerated") - .add("dimMultivalEnumerated2") - .add("dimMultivalSequentialWithNulls") - .add("dimSequential") - .add("dimSequentialHalfNull") - .add("dimUniform") - .add("dimZipf") - .add("metFloatNormal") - .add("metFloatZipf") - .add("metLongSequential") - .add("metLongUniform") - .build() + .add("dimMultivalEnumerated") + .add("dimMultivalEnumerated2") + .add("dimMultivalSequentialWithNulls") + .add("dimSequential") + .add("dimSequentialHalfNull") + .add("dimUniform") + .add("dimZipf") + .add("metFloatNormal") + .add("metFloatZipf") + .add("metLongSequential") + .add("metLongUniform") + .build() ) ) ) @@ -223,111 +243,111 @@ public Optional build( public static final List> RAW_ROWS1 = ImmutableList.of( ImmutableMap.builder() - .put("t", "2000-01-01") - .put("m1", "1.0") - .put("m2", "1.0") - .put("dim1", "") - .put("dim2", ImmutableList.of("a")) - .put("dim3", ImmutableList.of("a", "b")) - .build(), + .put("t", "2000-01-01") + .put("m1", "1.0") + .put("m2", "1.0") + .put("dim1", "") + .put("dim2", ImmutableList.of("a")) + .put("dim3", ImmutableList.of("a", "b")) + .build(), ImmutableMap.builder() - .put("t", "2000-01-02") - .put("m1", "2.0") - .put("m2", "2.0") - .put("dim1", "10.1") - .put("dim2", ImmutableList.of()) - .put("dim3", ImmutableList.of("b", "c")) - .build(), + .put("t", "2000-01-02") + .put("m1", "2.0") + .put("m2", "2.0") + .put("dim1", "10.1") + .put("dim2", ImmutableList.of()) + .put("dim3", ImmutableList.of("b", "c")) + .build(), ImmutableMap.builder() - .put("t", "2000-01-03") - .put("m1", "3.0") - .put("m2", "3.0") - .put("dim1", "2") - .put("dim2", ImmutableList.of("")) - .put("dim3", ImmutableList.of("d")) - .build(), + .put("t", "2000-01-03") + .put("m1", "3.0") + .put("m2", "3.0") + .put("dim1", "2") + .put("dim2", ImmutableList.of("")) + .put("dim3", ImmutableList.of("d")) + .build(), ImmutableMap.builder() - .put("t", "2001-01-01") - .put("m1", "4.0") - .put("m2", "4.0") - .put("dim1", "1") - .put("dim2", ImmutableList.of("a")) - .put("dim3", ImmutableList.of("")) - .build(), + .put("t", "2001-01-01") + .put("m1", "4.0") + .put("m2", "4.0") + .put("dim1", "1") + .put("dim2", ImmutableList.of("a")) + .put("dim3", ImmutableList.of("")) + .build(), ImmutableMap.builder() - .put("t", "2001-01-02") - .put("m1", "5.0") - .put("m2", "5.0") - .put("dim1", "def") - .put("dim2", ImmutableList.of("abc")) - .put("dim3", ImmutableList.of()) - .build(), + .put("t", "2001-01-02") + .put("m1", "5.0") + .put("m2", "5.0") + .put("dim1", "def") + .put("dim2", ImmutableList.of("abc")) + .put("dim3", ImmutableList.of()) + .build(), ImmutableMap.builder() - .put("t", "2001-01-03") - .put("m1", "6.0") - .put("m2", "6.0") - .put("dim1", "abc") - .build() + .put("t", "2001-01-03") + .put("m1", "6.0") + .put("m2", "6.0") + .put("dim1", "abc") + .build() ); public static final List RAW_ROWS1_X = ImmutableList.of( createRow( ImmutableMap.builder() - .put("t", "2000-01-01") - .put("m1_x", "1.0") - .put("m2_x", "1.0") - .put("dim1_x", "") - .put("dim2_x", ImmutableList.of("a")) - .put("dim3_x", ImmutableList.of("a", "b")) - .build() + .put("t", "2000-01-01") + .put("m1_x", "1.0") + .put("m2_x", "1.0") + .put("dim1_x", "") + .put("dim2_x", ImmutableList.of("a")) + .put("dim3_x", ImmutableList.of("a", "b")) + .build() ), createRow( ImmutableMap.builder() - .put("t", "2000-01-02") - .put("m1_x", "2.0") - .put("m2_x", "2.0") - .put("dim1_x", "10.1") - .put("dim2_x", ImmutableList.of()) - .put("dim3_x", ImmutableList.of("b", "c")) - .build() + .put("t", "2000-01-02") + .put("m1_x", "2.0") + .put("m2_x", "2.0") + .put("dim1_x", "10.1") + .put("dim2_x", ImmutableList.of()) + .put("dim3_x", ImmutableList.of("b", "c")) + .build() ), createRow( ImmutableMap.builder() - .put("t", "2000-01-03") - .put("m1_x", "3.0") - .put("m2_x", "3.0") - .put("dim1_x", "2") - .put("dim2_x", ImmutableList.of("")) - .put("dim3_x", ImmutableList.of("d")) - .build() + .put("t", "2000-01-03") + .put("m1_x", "3.0") + .put("m2_x", "3.0") + .put("dim1_x", "2") + .put("dim2_x", ImmutableList.of("")) + .put("dim3_x", ImmutableList.of("d")) + .build() ), createRow( ImmutableMap.builder() - .put("t", "2001-01-01") - .put("m1_x", "4.0") - .put("m2_x", "4.0") - .put("dim1_x", "1") - .put("dim2_x", ImmutableList.of("a")) - .put("dim3_x", ImmutableList.of("")) - .build() + .put("t", "2001-01-01") + .put("m1_x", "4.0") + .put("m2_x", "4.0") + .put("dim1_x", "1") + .put("dim2_x", ImmutableList.of("a")) + .put("dim3_x", ImmutableList.of("")) + .build() ), createRow( ImmutableMap.builder() - .put("t", "2001-01-02") - .put("m1_x", "5.0") - .put("m2_x", "5.0") - .put("dim1_x", "def") - .put("dim2_x", ImmutableList.of("abc")) - .put("dim3_x", ImmutableList.of()) - .build() + .put("t", "2001-01-02") + .put("m1_x", "5.0") + .put("m2_x", "5.0") + .put("dim1_x", "def") + .put("dim2_x", ImmutableList.of("abc")) + .put("dim3_x", ImmutableList.of()) + .build() ), createRow( ImmutableMap.builder() - .put("t", "2001-01-03") - .put("m1_x", "6.0") - .put("m2_x", "6.0") - .put("dim1_x", "abc") - .build() + .put("t", "2001-01-03") + .put("m1_x", "6.0") + .put("m2_x", "6.0") + .put("dim1_x", "abc") + .build() ) ); @@ -336,131 +356,131 @@ public Optional build( public static final List> RAW_ROWS1_WITH_NUMERIC_DIMS = ImmutableList.of( ImmutableMap.builder() - .put("t", "2000-01-01") - .put("m1", "1.0") - .put("m2", "1.0") - .put("d1", 1.0) - .put("f1", 1.0f) - .put("l1", 7L) - .put("dim1", "") - .put("dim2", ImmutableList.of("a")) - .put("dim3", ImmutableList.of("a", "b")) - .put("dim4", "a") - .put("dim5", "aa") - .put("dim6", "1") - .build(), + .put("t", "2000-01-01") + .put("m1", "1.0") + .put("m2", "1.0") + .put("d1", 1.0) + .put("f1", 1.0f) + .put("l1", 7L) + .put("dim1", "") + .put("dim2", ImmutableList.of("a")) + .put("dim3", ImmutableList.of("a", "b")) + .put("dim4", "a") + .put("dim5", "aa") + .put("dim6", "1") + .build(), ImmutableMap.builder() - .put("t", "2000-01-02") - .put("m1", "2.0") - .put("m2", "2.0") - .put("d1", 1.7) - .put("d2", 1.7) - .put("f1", 0.1f) - .put("f2", 0.1f) - .put("l1", 325323L) - .put("l2", 325323L) - .put("dim1", "10.1") - .put("dim2", ImmutableList.of()) - .put("dim3", ImmutableList.of("b", "c")) - .put("dim4", "a") - .put("dim5", "ab") - .put("dim6", "2") - .build(), + .put("t", "2000-01-02") + .put("m1", "2.0") + .put("m2", "2.0") + .put("d1", 1.7) + .put("d2", 1.7) + .put("f1", 0.1f) + .put("f2", 0.1f) + .put("l1", 325323L) + .put("l2", 325323L) + .put("dim1", "10.1") + .put("dim2", ImmutableList.of()) + .put("dim3", ImmutableList.of("b", "c")) + .put("dim4", "a") + .put("dim5", "ab") + .put("dim6", "2") + .build(), ImmutableMap.builder() - .put("t", "2000-01-03") - .put("m1", "3.0") - .put("m2", "3.0") - .put("d1", 0.0) - .put("d2", 0.0) - .put("f1", 0.0) - .put("f2", 0.0) - .put("l1", 0) - .put("l2", 0) - .put("dim1", "2") - .put("dim2", ImmutableList.of("")) - .put("dim3", ImmutableList.of("d")) - .put("dim4", "a") - .put("dim5", "ba") - .put("dim6", "3") - .build(), + .put("t", "2000-01-03") + .put("m1", "3.0") + .put("m2", "3.0") + .put("d1", 0.0) + .put("d2", 0.0) + .put("f1", 0.0) + .put("f2", 0.0) + .put("l1", 0) + .put("l2", 0) + .put("dim1", "2") + .put("dim2", ImmutableList.of("")) + .put("dim3", ImmutableList.of("d")) + .put("dim4", "a") + .put("dim5", "ba") + .put("dim6", "3") + .build(), ImmutableMap.builder() - .put("t", "2001-01-01") - .put("m1", "4.0") - .put("m2", "4.0") - .put("dim1", "1") - .put("dim2", ImmutableList.of("a")) - .put("dim3", ImmutableList.of("")) - .put("dim4", "b") - .put("dim5", "ad") - .put("dim6", "4") - .build(), + .put("t", "2001-01-01") + .put("m1", "4.0") + .put("m2", "4.0") + .put("dim1", "1") + .put("dim2", ImmutableList.of("a")) + .put("dim3", ImmutableList.of("")) + .put("dim4", "b") + .put("dim5", "ad") + .put("dim6", "4") + .build(), ImmutableMap.builder() - .put("t", "2001-01-02") - .put("m1", "5.0") - .put("m2", "5.0") - .put("dim1", "def") - .put("dim2", ImmutableList.of("abc")) - .put("dim3", ImmutableList.of()) - .put("dim4", "b") - .put("dim5", "aa") - .put("dim6", "5") - .build(), + .put("t", "2001-01-02") + .put("m1", "5.0") + .put("m2", "5.0") + .put("dim1", "def") + .put("dim2", ImmutableList.of("abc")) + .put("dim3", ImmutableList.of()) + .put("dim4", "b") + .put("dim5", "aa") + .put("dim6", "5") + .build(), ImmutableMap.builder() - .put("t", "2001-01-03") - .put("m1", "6.0") - .put("m2", "6.0") - .put("dim1", "abc") - .put("dim4", "b") - .put("dim5", "ab") - .put("dim6", "6") - .build() + .put("t", "2001-01-03") + .put("m1", "6.0") + .put("m2", "6.0") + .put("dim1", "abc") + .put("dim4", "b") + .put("dim5", "ab") + .put("dim6", "6") + .build() ); public static final List ROWS1_WITH_NUMERIC_DIMS = RAW_ROWS1_WITH_NUMERIC_DIMS.stream().map(raw -> createRow(raw, PARSER_NUMERIC_DIMS)).collect(Collectors.toList()); public static final List> RAW_ROWS2 = ImmutableList.of( ImmutableMap.builder() - .put("t", "2000-01-01") - .put("dim1", "דרואיד") - .put("dim2", "he") - .put("dim3", 10L) - .put("m1", 1.0) - .build(), + .put("t", "2000-01-01") + .put("dim1", "דרואיד") + .put("dim2", "he") + .put("dim3", 10L) + .put("m1", 1.0) + .build(), ImmutableMap.builder() - .put("t", "2000-01-01") - .put("dim1", "druid") - .put("dim2", "en") - .put("dim3", 11L) - .put("m1", 1.0) - .build(), + .put("t", "2000-01-01") + .put("dim1", "druid") + .put("dim2", "en") + .put("dim3", 11L) + .put("m1", 1.0) + .build(), ImmutableMap.builder() - .put("t", "2000-01-01") - .put("dim1", "друид") - .put("dim2", "ru") - .put("dim3", 12L) - .put("m1", 1.0) - .build() + .put("t", "2000-01-01") + .put("dim1", "друид") + .put("dim2", "ru") + .put("dim3", 12L) + .put("m1", 1.0) + .build() ); public static final List ROWS2 = RAW_ROWS2.stream().map(TestDataBuilder::createRow).collect(Collectors.toList()); public static final List> RAW_ROWS1_WITH_FULL_TIMESTAMP = ImmutableList.of( ImmutableMap.builder() - .put("t", "2000-01-01T10:51:45.695Z") - .put("m1", "1.0") - .put("m2", "1.0") - .put("dim1", "") - .put("dim2", ImmutableList.of("a")) - .put("dim3", ImmutableList.of("a", "b")) - .build(), + .put("t", "2000-01-01T10:51:45.695Z") + .put("m1", "1.0") + .put("m2", "1.0") + .put("dim1", "") + .put("dim2", ImmutableList.of("a")) + .put("dim3", ImmutableList.of("a", "b")) + .build(), ImmutableMap.builder() - .put("t", "2000-01-18T10:51:45.695Z") - .put("m1", "2.0") - .put("m2", "2.0") - .put("dim1", "10.1") - .put("dim2", ImmutableList.of()) - .put("dim3", ImmutableList.of("b", "c")) - .build() + .put("t", "2000-01-18T10:51:45.695Z") + .put("m1", "2.0") + .put("m2", "2.0") + .put("dim1", "10.1") + .put("dim2", ImmutableList.of()) + .put("dim3", ImmutableList.of("b", "c")) + .build() ); public static final List ROWS1_WITH_FULL_TIMESTAMP = RAW_ROWS1_WITH_FULL_TIMESTAMP.stream().map(TestDataBuilder::createRow).collect(Collectors.toList()); @@ -475,36 +495,36 @@ public Optional build( public static final List ROWS_LOTS_OF_COLUMNS = ImmutableList.of( createRow( ImmutableMap.builder() - .put("timestamp", 1576306800000L) - .put("metFloatZipf", 147.0) - .put("dimMultivalSequentialWithNulls", Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8")) - .put("dimMultivalEnumerated2", Arrays.asList(null, "Orange", "Apple")) - .put("metLongUniform", 372) - .put("metFloatNormal", 5000.0) - .put("dimZipf", "27") - .put("dimUniform", "74416") - .put("dimMultivalEnumerated", Arrays.asList("Baz", "World", "Hello", "Baz")) - .put("metLongSequential", 0) - .put("dimHyperUnique", "0") - .put("dimSequential", "0") - .put("dimSequentialHalfNull", "0") - .build(), + .put("timestamp", 1576306800000L) + .put("metFloatZipf", 147.0) + .put("dimMultivalSequentialWithNulls", Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8")) + .put("dimMultivalEnumerated2", Arrays.asList(null, "Orange", "Apple")) + .put("metLongUniform", 372) + .put("metFloatNormal", 5000.0) + .put("dimZipf", "27") + .put("dimUniform", "74416") + .put("dimMultivalEnumerated", Arrays.asList("Baz", "World", "Hello", "Baz")) + .put("metLongSequential", 0) + .put("dimHyperUnique", "0") + .put("dimSequential", "0") + .put("dimSequentialHalfNull", "0") + .build(), PARSER_LOTS_OF_COLUMNS ), createRow( ImmutableMap.builder() - .put("timestamp", 1576306800000L) - .put("metFloatZipf", 25.0) - .put("dimMultivalEnumerated2", Arrays.asList("Xylophone", null, "Corundum")) - .put("metLongUniform", 252) - .put("metFloatNormal", 4999.0) - .put("dimZipf", "9") - .put("dimUniform", "50515") - .put("dimMultivalEnumerated", Arrays.asList("Baz", "World", "ㅑ ㅓ ㅕ ㅗ ㅛ ㅜ ㅠ ㅡ ㅣ")) - .put("metLongSequential", 8) - .put("dimHyperUnique", "8") - .put("dimSequential", "8") - .build(), + .put("timestamp", 1576306800000L) + .put("metFloatZipf", 25.0) + .put("dimMultivalEnumerated2", Arrays.asList("Xylophone", null, "Corundum")) + .put("metLongUniform", 252) + .put("metFloatNormal", 4999.0) + .put("dimZipf", "9") + .put("dimUniform", "50515") + .put("dimMultivalEnumerated", Arrays.asList("Baz", "World", "ㅑ ㅓ ㅕ ㅗ ㅛ ㅜ ㅠ ㅡ ㅣ")) + .put("metLongSequential", 8) + .put("dimHyperUnique", "8") + .put("dimSequential", "8") + .build(), PARSER_LOTS_OF_COLUMNS ) ); @@ -575,6 +595,88 @@ public Optional build( DateTimes.nowUtc().toString() ); + public static QueryableIndex makeWikipediaIndex(File tmpDir) + { + final List dimensions = Arrays.asList( + new StringDimensionSchema("channel"), + new StringDimensionSchema("cityName"), + new StringDimensionSchema("comment"), + new StringDimensionSchema("countryIsoCode"), + new StringDimensionSchema("countryName"), + new StringDimensionSchema("isAnonymous"), + new StringDimensionSchema("isMinor"), + new StringDimensionSchema("isNew"), + new StringDimensionSchema("isRobot"), + new StringDimensionSchema("isUnpatrolled"), + new StringDimensionSchema("metroCode"), + new StringDimensionSchema("namespace"), + new StringDimensionSchema("page"), + new StringDimensionSchema("regionIsoCode"), + new StringDimensionSchema("regionName"), + new StringDimensionSchema("user"), + new LongDimensionSchema("delta"), + new LongDimensionSchema("added"), + new LongDimensionSchema("deleted") + ); + ArrayList dimensionNames = new ArrayList<>(dimensions.size()); + for (DimensionSchema dimension : dimensions) { + dimensionNames.add(dimension.getName()); + } + + return IndexBuilder + .create() + .tmpDir(new File(tmpDir, "wikipedia1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema(new IncrementalIndexSchema.Builder() + .withRollup(false) + .withDimensionsSpec(new DimensionsSpec(dimensions)) + .build() + ) + .rows( + () -> { + final InputStream is; + try { + is = new GZIPInputStream( + // The extension ".json.gz" appears to not be included in resource bundles, so name it ".jgz"! + ClassLoader.getSystemResourceAsStream("calcite/tests/wikiticker-2015-09-12-sampled.jgz") + ); + } + catch (IOException e) { + throw new RE(e, "problem loading wikipedia dataset for tests"); + } + + ObjectMapper mapper = new DefaultObjectMapper(); + + // This method is returning an iterator over a BufferedReader, attempts are made to try to close the reader if + // exceptions occur, but this is happening in test setup and failures here should generally fail the tests, so + // leaks are not a primary concern. If anything were to actually try to mimic this code in real life, it should + // do a better job of taking care of resources. + BufferedReader lines = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); + return lines + .lines() + .map(line -> { + try { + Map map = mapper.readValue(line, Map.class); + final String time = String.valueOf(map.get("time")); + return (InputRow) new MapBasedInputRow(DateTimes.of(time), dimensionNames, map); + } + catch (JsonProcessingException e) { + final RE toThrow = new RE(e, "Problem reading line setting up wikipedia dataset for tests."); + try { + is.close(); + } + catch (IOException logged) { + toThrow.addSuppressed(logged); + } + throw toThrow; + } + }) + .iterator(); + } + ) + .buildMMappedIndex(); + } + public static SpecificSegmentsQuerySegmentWalker createMockWalker( final Injector injector, final QueryRunnerFactoryConglomerate conglomerate, @@ -712,7 +814,6 @@ public static SpecificSegmentsQuerySegmentWalker createMockWalker( .rows(USER_VISIT_ROWS) .buildMMappedIndex(); - return new SpecificSegmentsQuerySegmentWalker( conglomerate, injector.getInstance(LookupExtractorFactoryContainerProvider.class), @@ -808,6 +909,15 @@ public static SpecificSegmentsQuerySegmentWalker createMockWalker( .size(0) .build(), userVisitIndex + ).add( + DataSegment.builder() + .dataSource("wikipedia") + .interval(Intervals.of("2015-09-12/2015-09-13")) + .version("1") + .shardSpec(new NumberedShardSpec(0, 0)) + .size(0) + .build(), + makeWikipediaIndex(tmpDir) ); } @@ -815,6 +925,7 @@ private static MapBasedInputRow toRow(String time, List dimensions, Map< { return new MapBasedInputRow(DateTimes.ISO_DATE_OPTIONAL_TIME.parse(time), dimensions, event); } + public static InputRow createRow(final ImmutableMap map) { return PARSER.parseBatch((Map) map).get(0); diff --git a/sql/src/test/resources/calcite/tests/wikiticker-2015-09-12-sampled.jgz b/sql/src/test/resources/calcite/tests/wikiticker-2015-09-12-sampled.jgz new file mode 100644 index 000000000000..6ca3233b777c Binary files /dev/null and b/sql/src/test/resources/calcite/tests/wikiticker-2015-09-12-sampled.jgz differ diff --git a/sql/src/test/resources/calcite/tests/window/simpleSum.sqlTest b/sql/src/test/resources/calcite/tests/window/simpleSum.sqlTest new file mode 100644 index 000000000000..d10e710c5d45 --- /dev/null +++ b/sql/src/test/resources/calcite/tests/window/simpleSum.sqlTest @@ -0,0 +1,25 @@ +type: "operatorValidation" + +sql: " + SELECT + FLOOR(__time TO DAY) t, + SUM(cnt) c, + SUM(SUM(cnt)) OVER (ORDER BY FLOOR(__time TO DAY)) cc + FROM foo + GROUP BY FLOOR(__time TO DAY)" + +expectedOperators: + - { type: "naivePartition", partitionColumns: [ ] } + - type: "window" + processor: + type: "aggregate" + cumulativeAggregations: + - { type: "longSum", name: "w0", fieldName: "a0" } + +expectedResults: + - [946684800000, 1, 1] + - [946771200000, 1, 2] + - [946857600000, 1, 3] + - [978307200000, 1, 4] + - [978393600000, 1, 5] + - [978480000000, 1, 6] diff --git a/sql/src/test/resources/calcite/tests/window/wikipediaCumulativeOrdered.sqlTest b/sql/src/test/resources/calcite/tests/window/wikipediaCumulativeOrdered.sqlTest new file mode 100644 index 000000000000..e87e1db391e0 --- /dev/null +++ b/sql/src/test/resources/calcite/tests/window/wikipediaCumulativeOrdered.sqlTest @@ -0,0 +1,1028 @@ +type: "operatorValidation" + +sql: " + SELECT + countryIsoCode, + FLOOR(__time TO HOUR) t, + SUM(delta) delta, + SUM(SUM(delta)) OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) totalDelta, + LAG(FLOOR(__time TO HOUR), 2) OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) laggardTime, + LEAD(FLOOR(__time TO HOUR), 1) OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) leadTime, + FIRST_VALUE(SUM(delta)) OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) AS firstDelay, + LAST_VALUE(SUM(delta)) OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) AS lastDelay, + NTILE(3) OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) AS delayNTile, + RANK() OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) AS delayRank, + PERCENT_RANK() OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) AS delayRankPercent, + DENSE_RANK() OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) AS delayRankDense, + CUME_DIST() OVER (PARTITION BY countryIsoCode ORDER BY SUM(delta)) AS delayCumeDist + FROM wikipedia + GROUP BY 1, 2 + ORDER BY 1, 3" + +expectedOperators: + - { type: "naivePartition", partitionColumns: ["d0"] } + - type: "window" + processor: + type: "composing" + processors: + - {"type":"offset", "inputColumn":"d1", "outputColumn":"w1", "offset":-2} + - {"type":"offset", "inputColumn":"d1", "outputColumn":"w2", "offset":1} + - {"type":"first", "inputColumn":"a0", "outputColumn":"w3"} + - {"type":"last", "inputColumn":"a0", "outputColumn":"w4"} + - {"type":"percentile", "outputColumn":"w5", "numBuckets":3} + - {"type":"rank", "group":["a0"], "outputColumn":"w6", "asPercent":false} + - {"type":"rank", "group":["a0"], "outputColumn":"w7", "asPercent":true} + - {"type":"denseRank", "group":["a0"], "outputColumn":"w8"} + - {"type":"cumeDist", "group":["a0"], "outputColumn":"w9"} + - type: "aggregate" + cumulativeAggregations: + - {"type":"longSum", "name":"w0", "fieldName":"a0"} + +expectedResults: + - ["", 1442016000000, 29873, 29873, null, 1442030400000, 29873, 787370, 1, 1, 0.0, 1, 0.041666666666666664] + - ["", 1442030400000, 166672, 196545, null, 1442019600000, 29873, 787370, 1, 2, 0.043478260869565216, 2, 0.08333333333333333] + - ["", 1442019600000, 173892, 370437, 1442016000000, 1442037600000, 29873, 787370, 1, 3, 0.08695652173913043, 3, 0.125] + - ["", 1442037600000, 200605, 571042, 1442030400000, 1442026800000, 29873, 787370, 1, 4, 0.13043478260869565, 4, 0.16666666666666666] + - ["", 1442026800000, 252626, 823668, 1442019600000, 1442098800000, 29873, 787370, 1, 5, 0.17391304347826086, 5, 0.20833333333333334] + - ["", 1442098800000, 276159, 1099827, 1442037600000, 1442055600000, 29873, 787370, 1, 6, 0.21739130434782608, 6, 0.25] + - ["", 1442055600000, 283958, 1383785, 1442026800000, 1442088000000, 29873, 787370, 1, 7, 0.2608695652173913, 7, 0.2916666666666667] + - ["", 1442088000000, 303872, 1687657, 1442098800000, 1442048400000, 29873, 787370, 1, 8, 0.30434782608695654, 8, 0.3333333333333333] + - ["", 1442048400000, 308316, 1995973, 1442055600000, 1442044800000, 29873, 787370, 2, 9, 0.34782608695652173, 9, 0.375] + - ["", 1442044800000, 316002, 2311975, 1442088000000, 1442034000000, 29873, 787370, 2, 10, 0.391304347826087, 10, 0.4166666666666667] + - ["", 1442034000000, 330957, 2642932, 1442048400000, 1442066400000, 29873, 787370, 2, 11, 0.43478260869565216, 11, 0.4583333333333333] + - ["", 1442066400000, 351584, 2994516, 1442044800000, 1442070000000, 29873, 787370, 2, 12, 0.4782608695652174, 12, 0.5] + - ["", 1442070000000, 358515, 3353031, 1442034000000, 1442084400000, 29873, 787370, 2, 13, 0.5217391304347826, 13, 0.5416666666666666] + - ["", 1442084400000, 372569, 3725600, 1442066400000, 1442095200000, 29873, 787370, 2, 14, 0.5652173913043478, 14, 0.5833333333333334] + - ["", 1442095200000, 374501, 4100101, 1442070000000, 1442073600000, 29873, 787370, 2, 15, 0.6086956521739131, 15, 0.625] + - ["", 1442073600000, 375394, 4475495, 1442084400000, 1442062800000, 29873, 787370, 2, 16, 0.6521739130434783, 16, 0.6666666666666666] + - ["", 1442062800000, 389465, 4864960, 1442095200000, 1442077200000, 29873, 787370, 3, 17, 0.6956521739130435, 17, 0.7083333333333334] + - ["", 1442077200000, 392483, 5257443, 1442073600000, 1442023200000, 29873, 787370, 3, 18, 0.7391304347826086, 18, 0.75] + - ["", 1442023200000, 399636, 5657079, 1442062800000, 1442080800000, 29873, 787370, 3, 19, 0.782608695652174, 19, 0.7916666666666666] + - ["", 1442080800000, 453077, 6110156, 1442077200000, 1442059200000, 29873, 787370, 3, 20, 0.8260869565217391, 20, 0.8333333333333334] + - ["", 1442059200000, 459297, 6569453, 1442023200000, 1442091600000, 29873, 787370, 3, 21, 0.8695652173913043, 21, 0.875] + - ["", 1442091600000, 514427, 7083880, 1442080800000, 1442041200000, 29873, 787370, 3, 22, 0.9130434782608695, 22, 0.9166666666666666] + - ["", 1442041200000, 543450, 7627330, 1442059200000, 1442052000000, 29873, 787370, 3, 23, 0.9565217391304348, 23, 0.9583333333333334] + - ["", 1442052000000, 787370, 8414700, 1442091600000, null, 29873, 787370, 3, 24, 1.0, 24, 1.0] + - ["AE", 1442059200000, -11, -11, null, 1442044800000, -11, 6323, 1, 1, 0.0, 1, 0.125] + - ["AE", 1442044800000, -7, -18, null, 1442052000000, -11, 6323, 1, 2, 0.14285714285714285, 2, 0.25] + - ["AE", 1442052000000, -3, -21, 1442059200000, 1442048400000, -11, 6323, 1, 3, 0.2857142857142857, 3, 0.375] + - ["AE", 1442048400000, 39, 18, 1442044800000, 1442080800000, -11, 6323, 2, 4, 0.42857142857142855, 4, 0.5] + - ["AE", 1442080800000, 42, 60, 1442052000000, 1442070000000, -11, 6323, 2, 5, 0.5714285714285714, 5, 0.625] + - ["AE", 1442070000000, 46, 106, 1442048400000, 1442030400000, -11, 6323, 2, 6, 0.7142857142857143, 6, 0.75] + - ["AE", 1442030400000, 118, 224, 1442080800000, 1442077200000, -11, 6323, 3, 7, 0.8571428571428571, 7, 0.875] + - ["AE", 1442077200000, 6323, 6547, 1442070000000, null, -11, 6323, 3, 8, 1.0, 8, 1.0] + - ["AL", 1442077200000, 26, 26, null, 1442091600000, 26, 54, 1, 1, 0.0, 1, 0.5] + - ["AL", 1442091600000, 54, 80, null, null, 26, 54, 2, 2, 1.0, 2, 1.0] + - ["AO", 1442041200000, -26, -26, null, 1442052000000, -26, 722, 1, 1, 0.0, 1, 0.25] + - ["AO", 1442052000000, -18, -44, null, 1442088000000, -26, 722, 1, 2, 0.3333333333333333, 2, 0.5] + - ["AO", 1442088000000, 62, 18, 1442041200000, 1442098800000, -26, 722, 2, 3, 0.6666666666666666, 3, 0.75] + - ["AO", 1442098800000, 722, 740, 1442052000000, null, -26, 722, 3, 4, 1.0, 4, 1.0] + - ["AR", 1442077200000, -591, -591, null, 1442055600000, -591, 2514, 1, 1, 0.0, 1, 0.058823529411764705] + - ["AR", 1442055600000, -54, -645, null, 1442084400000, -591, 2514, 1, 2, 0.0625, 2, 0.11764705882352941] + - ["AR", 1442084400000, -5, -650, 1442077200000, 1442030400000, -591, 2514, 1, 3, 0.125, 3, 0.17647058823529413] + - ["AR", 1442030400000, -3, -653, 1442055600000, 1442066400000, -591, 2514, 1, 4, 0.1875, 4, 0.23529411764705882] + - ["AR", 1442066400000, 0, -653, 1442084400000, 1442019600000, -591, 2514, 1, 5, 0.25, 5, 0.29411764705882354] + - ["AR", 1442019600000, 1, -652, 1442030400000, 1442080800000, -591, 2514, 1, 6, 0.3125, 6, 0.4117647058823529] + - ["AR", 1442080800000, 1, -651, 1442066400000, 1442062800000, -591, 2514, 2, 6, 0.3125, 6, 0.4117647058823529] + - ["AR", 1442062800000, 29, -622, 1442019600000, 1442098800000, -591, 2514, 2, 8, 0.4375, 7, 0.47058823529411764] + - ["AR", 1442098800000, 64, -558, 1442080800000, 1442037600000, -591, 2514, 2, 9, 0.5, 8, 0.5294117647058824] + - ["AR", 1442037600000, 81, -477, 1442062800000, 1442059200000, -591, 2514, 2, 10, 0.5625, 9, 0.5882352941176471] + - ["AR", 1442059200000, 210, -267, 1442098800000, 1442034000000, -591, 2514, 2, 11, 0.625, 10, 0.6470588235294118] + - ["AR", 1442034000000, 212, -55, 1442037600000, 1442091600000, -591, 2514, 2, 12, 0.6875, 11, 0.7058823529411765] + - ["AR", 1442091600000, 340, 285, 1442059200000, 1442070000000, -591, 2514, 3, 13, 0.75, 12, 0.7647058823529411] + - ["AR", 1442070000000, 377, 662, 1442034000000, 1442095200000, -591, 2514, 3, 14, 0.8125, 13, 0.8235294117647058] + - ["AR", 1442095200000, 630, 1292, 1442091600000, 1442026800000, -591, 2514, 3, 15, 0.875, 14, 0.8823529411764706] + - ["AR", 1442026800000, 644, 1936, 1442070000000, 1442023200000, -591, 2514, 3, 16, 0.9375, 15, 0.9411764705882353] + - ["AR", 1442023200000, 2514, 4450, 1442095200000, null, -591, 2514, 3, 17, 1.0, 16, 1.0] + - ["AT", 1442062800000, -155, -155, null, 1442084400000, -155, 7050, 1, 1, 0.0, 1, 0.14285714285714285] + - ["AT", 1442084400000, -2, -157, null, 1442066400000, -155, 7050, 1, 2, 0.16666666666666666, 2, 0.2857142857142857] + - ["AT", 1442066400000, 0, -157, 1442062800000, 1442091600000, -155, 7050, 1, 3, 0.3333333333333333, 3, 0.42857142857142855] + - ["AT", 1442091600000, 89, -68, 1442084400000, 1442070000000, -155, 7050, 2, 4, 0.5, 4, 0.5714285714285714] + - ["AT", 1442070000000, 272, 204, 1442066400000, 1442052000000, -155, 7050, 2, 5, 0.6666666666666666, 5, 0.7142857142857143] + - ["AT", 1442052000000, 4793, 4997, 1442091600000, 1442088000000, -155, 7050, 3, 6, 0.8333333333333334, 6, 0.8571428571428571] + - ["AT", 1442088000000, 7050, 12047, 1442070000000, null, -155, 7050, 3, 7, 1.0, 7, 1.0] + - ["AU", 1442052000000, -643, -643, null, 1442030400000, -643, 1138, 1, 1, 0.0, 1, 0.05263157894736842] + - ["AU", 1442030400000, -377, -1020, null, 1442066400000, -643, 1138, 1, 2, 0.05555555555555555, 2, 0.10526315789473684] + - ["AU", 1442066400000, -21, -1041, 1442052000000, 1442070000000, -643, 1138, 1, 3, 0.1111111111111111, 3, 0.15789473684210525] + - ["AU", 1442070000000, -12, -1053, 1442030400000, 1442016000000, -643, 1138, 1, 4, 0.16666666666666666, 4, 0.21052631578947367] + - ["AU", 1442016000000, 0, -1053, 1442066400000, 1442077200000, -643, 1138, 1, 5, 0.2222222222222222, 5, 0.2631578947368421] + - ["AU", 1442077200000, 1, -1052, 1442070000000, 1442037600000, -643, 1138, 1, 6, 0.2777777777777778, 6, 0.3157894736842105] + - ["AU", 1442037600000, 3, -1049, 1442016000000, 1442059200000, -643, 1138, 1, 7, 0.3333333333333333, 7, 0.3684210526315789] + - ["AU", 1442059200000, 38, -1011, 1442077200000, 1442023200000, -643, 1138, 2, 8, 0.3888888888888889, 8, 0.42105263157894735] + - ["AU", 1442023200000, 52, -959, 1442037600000, 1442048400000, -643, 1138, 2, 9, 0.4444444444444444, 9, 0.47368421052631576] + - ["AU", 1442048400000, 135, -824, 1442059200000, 1442055600000, -643, 1138, 2, 10, 0.5, 10, 0.5263157894736842] + - ["AU", 1442055600000, 182, -642, 1442023200000, 1442026800000, -643, 1138, 2, 11, 0.5555555555555556, 11, 0.5789473684210527] + - ["AU", 1442026800000, 188, -454, 1442048400000, 1442041200000, -643, 1138, 2, 12, 0.6111111111111112, 12, 0.631578947368421] + - ["AU", 1442041200000, 194, -260, 1442055600000, 1442019600000, -643, 1138, 2, 13, 0.6666666666666666, 13, 0.6842105263157895] + - ["AU", 1442019600000, 253, -7, 1442026800000, 1442034000000, -643, 1138, 3, 14, 0.7222222222222222, 14, 0.7368421052631579] + - ["AU", 1442034000000, 283, 276, 1442041200000, 1442044800000, -643, 1138, 3, 15, 0.7777777777777778, 15, 0.7894736842105263] + - ["AU", 1442044800000, 373, 649, 1442019600000, 1442095200000, -643, 1138, 3, 16, 0.8333333333333334, 16, 0.8421052631578947] + - ["AU", 1442095200000, 395, 1044, 1442034000000, 1442098800000, -643, 1138, 3, 17, 0.8888888888888888, 17, 0.8947368421052632] + - ["AU", 1442098800000, 518, 1562, 1442044800000, 1442091600000, -643, 1138, 3, 18, 0.9444444444444444, 18, 0.9473684210526315] + - ["AU", 1442091600000, 1138, 2700, 1442095200000, null, -643, 1138, 3, 19, 1.0, 19, 1.0] + - ["BA", 1442055600000, -202, -202, null, 1442048400000, -202, 38, 1, 1, 0.0, 1, 0.25] + - ["BA", 1442048400000, -13, -215, null, 1442084400000, -202, 38, 1, 2, 0.3333333333333333, 2, 0.5] + - ["BA", 1442084400000, -1, -216, 1442055600000, 1442052000000, -202, 38, 2, 3, 0.6666666666666666, 3, 0.75] + - ["BA", 1442052000000, 38, -178, 1442048400000, null, -202, 38, 3, 4, 1.0, 4, 1.0] + - ["BD", 1442091600000, -2, -2, null, 1442019600000, -2, 854, 1, 1, 0.0, 1, 0.16666666666666666] + - ["BD", 1442019600000, 0, -2, null, 1442077200000, -2, 854, 1, 2, 0.2, 2, 0.3333333333333333] + - ["BD", 1442077200000, 75, 73, 1442091600000, 1442066400000, -2, 854, 2, 3, 0.4, 3, 0.5] + - ["BD", 1442066400000, 76, 149, 1442019600000, 1442073600000, -2, 854, 2, 4, 0.6, 4, 0.6666666666666666] + - ["BD", 1442073600000, 103, 252, 1442077200000, 1442041200000, -2, 854, 3, 5, 0.8, 5, 0.8333333333333334] + - ["BD", 1442041200000, 854, 1106, 1442066400000, null, -2, 854, 3, 6, 1.0, 6, 1.0] + - ["BE", 1442030400000, -103, -103, null, 1442052000000, -103, 233, 1, 1, 0.0, 1, 0.08333333333333333] + - ["BE", 1442052000000, -1, -104, null, 1442080800000, -103, 233, 1, 2, 0.09090909090909091, 2, 0.16666666666666666] + - ["BE", 1442080800000, 1, -103, 1442030400000, 1442098800000, -103, 233, 1, 3, 0.18181818181818182, 3, 0.25] + - ["BE", 1442098800000, 9, -94, 1442052000000, 1442073600000, -103, 233, 1, 4, 0.2727272727272727, 4, 0.3333333333333333] + - ["BE", 1442073600000, 19, -75, 1442080800000, 1442048400000, -103, 233, 2, 5, 0.36363636363636365, 5, 0.4166666666666667] + - ["BE", 1442048400000, 59, -16, 1442098800000, 1442088000000, -103, 233, 2, 6, 0.45454545454545453, 6, 0.5] + - ["BE", 1442088000000, 67, 51, 1442073600000, 1442062800000, -103, 233, 2, 7, 0.5454545454545454, 7, 0.5833333333333334] + - ["BE", 1442062800000, 91, 142, 1442048400000, 1442091600000, -103, 233, 2, 8, 0.6363636363636364, 8, 0.6666666666666666] + - ["BE", 1442091600000, 101, 243, 1442088000000, 1442066400000, -103, 233, 3, 9, 0.7272727272727273, 9, 0.75] + - ["BE", 1442066400000, 136, 379, 1442062800000, 1442084400000, -103, 233, 3, 10, 0.8181818181818182, 10, 0.8333333333333334] + - ["BE", 1442084400000, 183, 562, 1442091600000, 1442055600000, -103, 233, 3, 11, 0.9090909090909091, 11, 0.9166666666666666] + - ["BE", 1442055600000, 233, 795, 1442066400000, null, -103, 233, 3, 12, 1.0, 12, 1.0] + - ["BG", 1442041200000, 9, 9, null, 1442070000000, 9, 18936, 1, 1, 0.0, 1, 0.2] + - ["BG", 1442070000000, 55, 64, null, 1442059200000, 9, 18936, 1, 2, 0.25, 2, 0.4] + - ["BG", 1442059200000, 191, 255, 1442041200000, 1442084400000, 9, 18936, 2, 3, 0.5, 3, 0.6] + - ["BG", 1442084400000, 401, 656, 1442070000000, 1442052000000, 9, 18936, 2, 4, 0.75, 4, 0.8] + - ["BG", 1442052000000, 18936, 19592, 1442059200000, null, 9, 18936, 3, 5, 1.0, 5, 1.0] + - ["BH", 1442052000000, 44, 44, null, null, 44, 44, 1, 1, 0.0, 1, 1.0] + - ["BO", 1442095200000, -4, -4, null, 1442080800000, -4, 4, 1, 1, 0.0, 1, 0.3333333333333333] + - ["BO", 1442080800000, 4, 0, null, 1442088000000, -4, 4, 2, 2, 0.5, 2, 1.0] + - ["BO", 1442088000000, 4, 4, 1442095200000, null, -4, 4, 3, 2, 0.5, 2, 1.0] + - ["BR", 1442098800000, -645, -645, null, 1442080800000, -645, 2253, 1, 1, 0.0, 1, 0.043478260869565216] + - ["BR", 1442080800000, -267, -912, null, 1442016000000, -645, 2253, 1, 2, 0.045454545454545456, 2, 0.08695652173913043] + - ["BR", 1442016000000, -248, -1160, 1442098800000, 1442041200000, -645, 2253, 1, 3, 0.09090909090909091, 3, 0.13043478260869565] + - ["BR", 1442041200000, 3, -1157, 1442080800000, 1442091600000, -645, 2253, 1, 4, 0.13636363636363635, 4, 0.17391304347826086] + - ["BR", 1442091600000, 11, -1146, 1442016000000, 1442034000000, -645, 2253, 1, 5, 0.18181818181818182, 5, 0.21739130434782608] + - ["BR", 1442034000000, 21, -1125, 1442041200000, 1442030400000, -645, 2253, 1, 6, 0.22727272727272727, 6, 0.2608695652173913] + - ["BR", 1442030400000, 30, -1095, 1442091600000, 1442026800000, -645, 2253, 1, 7, 0.2727272727272727, 7, 0.30434782608695654] + - ["BR", 1442026800000, 51, -1044, 1442034000000, 1442044800000, -645, 2253, 1, 8, 0.3181818181818182, 8, 0.34782608695652173] + - ["BR", 1442044800000, 71, -973, 1442030400000, 1442059200000, -645, 2253, 2, 9, 0.36363636363636365, 9, 0.391304347826087] + - ["BR", 1442059200000, 73, -900, 1442026800000, 1442062800000, -645, 2253, 2, 10, 0.4090909090909091, 10, 0.43478260869565216] + - ["BR", 1442062800000, 93, -807, 1442044800000, 1442088000000, -645, 2253, 2, 11, 0.45454545454545453, 11, 0.4782608695652174] + - ["BR", 1442088000000, 215, -592, 1442059200000, 1442052000000, -645, 2253, 2, 12, 0.5, 12, 0.5217391304347826] + - ["BR", 1442052000000, 232, -360, 1442062800000, 1442055600000, -645, 2253, 2, 13, 0.5454545454545454, 13, 0.5652173913043478] + - ["BR", 1442055600000, 242, -118, 1442088000000, 1442037600000, -645, 2253, 2, 14, 0.5909090909090909, 14, 0.6086956521739131] + - ["BR", 1442037600000, 267, 149, 1442052000000, 1442019600000, -645, 2253, 2, 15, 0.6363636363636364, 15, 0.6521739130434783] + - ["BR", 1442019600000, 372, 521, 1442055600000, 1442084400000, -645, 2253, 2, 16, 0.6818181818181818, 16, 0.6956521739130435] + - ["BR", 1442084400000, 492, 1013, 1442037600000, 1442070000000, -645, 2253, 3, 17, 0.7272727272727273, 17, 0.7391304347826086] + - ["BR", 1442070000000, 536, 1549, 1442019600000, 1442095200000, -645, 2253, 3, 18, 0.7727272727272727, 18, 0.782608695652174] + - ["BR", 1442095200000, 748, 2297, 1442084400000, 1442023200000, -645, 2253, 3, 19, 0.8181818181818182, 19, 0.8260869565217391] + - ["BR", 1442023200000, 879, 3176, 1442070000000, 1442066400000, -645, 2253, 3, 20, 0.8636363636363636, 20, 0.8695652173913043] + - ["BR", 1442066400000, 1034, 4210, 1442095200000, 1442073600000, -645, 2253, 3, 21, 0.9090909090909091, 21, 0.9130434782608695] + - ["BR", 1442073600000, 2087, 6297, 1442023200000, 1442077200000, -645, 2253, 3, 22, 0.9545454545454546, 22, 0.9565217391304348] + - ["BR", 1442077200000, 2253, 8550, 1442066400000, null, -645, 2253, 3, 23, 1.0, 23, 1.0] + - ["BY", 1442055600000, 1, 1, null, 1442084400000, 1, 1464, 1, 1, 0.0, 1, 0.2857142857142857] + - ["BY", 1442084400000, 1, 2, null, 1442080800000, 1, 1464, 1, 1, 0.0, 1, 0.2857142857142857] + - ["BY", 1442080800000, 28, 30, 1442055600000, 1442077200000, 1, 1464, 1, 3, 0.3333333333333333, 2, 0.42857142857142855] + - ["BY", 1442077200000, 30, 60, 1442084400000, 1442088000000, 1, 1464, 2, 4, 0.5, 3, 0.5714285714285714] + - ["BY", 1442088000000, 33, 93, 1442080800000, 1442073600000, 1, 1464, 2, 5, 0.6666666666666666, 4, 0.7142857142857143] + - ["BY", 1442073600000, 596, 689, 1442077200000, 1442059200000, 1, 1464, 3, 6, 0.8333333333333334, 5, 0.8571428571428571] + - ["BY", 1442059200000, 1464, 2153, 1442088000000, null, 1, 1464, 3, 7, 1.0, 6, 1.0] + - ["CA", 1442016000000, -371, -371, null, 1442062800000, -371, 2858, 1, 1, 0.0, 1, 0.045454545454545456] + - ["CA", 1442062800000, -367, -738, null, 1442095200000, -371, 2858, 1, 2, 0.047619047619047616, 2, 0.09090909090909091] + - ["CA", 1442095200000, -361, -1099, 1442016000000, 1442077200000, -371, 2858, 1, 3, 0.09523809523809523, 3, 0.13636363636363635] + - ["CA", 1442077200000, -282, -1381, 1442062800000, 1442037600000, -371, 2858, 1, 4, 0.14285714285714285, 4, 0.18181818181818182] + - ["CA", 1442037600000, -132, -1513, 1442095200000, 1442030400000, -371, 2858, 1, 5, 0.19047619047619047, 5, 0.22727272727272727] + - ["CA", 1442030400000, -47, -1560, 1442077200000, 1442044800000, -371, 2858, 1, 6, 0.23809523809523808, 6, 0.2727272727272727] + - ["CA", 1442044800000, 1, -1559, 1442037600000, 1442041200000, -371, 2858, 1, 7, 0.2857142857142857, 7, 0.3181818181818182] + - ["CA", 1442041200000, 5, -1554, 1442030400000, 1442088000000, -371, 2858, 1, 8, 0.3333333333333333, 8, 0.36363636363636365] + - ["CA", 1442088000000, 35, -1519, 1442044800000, 1442052000000, -371, 2858, 2, 9, 0.38095238095238093, 9, 0.4090909090909091] + - ["CA", 1442052000000, 38, -1481, 1442041200000, 1442084400000, -371, 2858, 2, 10, 0.42857142857142855, 10, 0.45454545454545453] + - ["CA", 1442084400000, 44, -1437, 1442088000000, 1442073600000, -371, 2858, 2, 11, 0.47619047619047616, 11, 0.5] + - ["CA", 1442073600000, 86, -1351, 1442052000000, 1442098800000, -371, 2858, 2, 12, 0.5238095238095238, 12, 0.5454545454545454] + - ["CA", 1442098800000, 164, -1187, 1442084400000, 1442034000000, -371, 2858, 2, 13, 0.5714285714285714, 13, 0.5909090909090909] + - ["CA", 1442034000000, 178, -1009, 1442073600000, 1442070000000, -371, 2858, 2, 14, 0.6190476190476191, 14, 0.6363636363636364] + - ["CA", 1442070000000, 185, -824, 1442098800000, 1442023200000, -371, 2858, 2, 15, 0.6666666666666666, 15, 0.6818181818181818] + - ["CA", 1442023200000, 286, -538, 1442034000000, 1442066400000, -371, 2858, 3, 16, 0.7142857142857143, 16, 0.7272727272727273] + - ["CA", 1442066400000, 307, -231, 1442070000000, 1442080800000, -371, 2858, 3, 17, 0.7619047619047619, 17, 0.7727272727272727] + - ["CA", 1442080800000, 481, 250, 1442023200000, 1442059200000, -371, 2858, 3, 18, 0.8095238095238095, 18, 0.8181818181818182] + - ["CA", 1442059200000, 1036, 1286, 1442066400000, 1442019600000, -371, 2858, 3, 19, 0.8571428571428571, 19, 0.8636363636363636] + - ["CA", 1442019600000, 2184, 3470, 1442080800000, 1442026800000, -371, 2858, 3, 20, 0.9047619047619048, 20, 0.9090909090909091] + - ["CA", 1442026800000, 2216, 5686, 1442059200000, 1442091600000, -371, 2858, 3, 21, 0.9523809523809523, 21, 0.9545454545454546] + - ["CA", 1442091600000, 2858, 8544, 1442019600000, null, -371, 2858, 3, 22, 1.0, 22, 1.0] + - ["CH", 1442044800000, -54, -54, null, 1442055600000, -54, 360, 1, 1, 0.0, 1, 0.08333333333333333] + - ["CH", 1442055600000, 0, -54, null, 1442077200000, -54, 360, 1, 2, 0.09090909090909091, 2, 0.16666666666666666] + - ["CH", 1442077200000, 6, -48, 1442044800000, 1442070000000, -54, 360, 1, 3, 0.18181818181818182, 3, 0.25] + - ["CH", 1442070000000, 11, -37, 1442055600000, 1442084400000, -54, 360, 1, 4, 0.2727272727272727, 4, 0.3333333333333333] + - ["CH", 1442084400000, 13, -24, 1442077200000, 1442062800000, -54, 360, 2, 5, 0.36363636363636365, 5, 0.4166666666666667] + - ["CH", 1442062800000, 22, -2, 1442070000000, 1442048400000, -54, 360, 2, 6, 0.45454545454545453, 6, 0.5] + - ["CH", 1442048400000, 24, 22, 1442084400000, 1442052000000, -54, 360, 2, 7, 0.5454545454545454, 7, 0.5833333333333334] + - ["CH", 1442052000000, 47, 69, 1442062800000, 1442037600000, -54, 360, 2, 8, 0.6363636363636364, 8, 0.6666666666666666] + - ["CH", 1442037600000, 59, 128, 1442048400000, 1442091600000, -54, 360, 3, 9, 0.7272727272727273, 9, 0.75] + - ["CH", 1442091600000, 67, 195, 1442052000000, 1442041200000, -54, 360, 3, 10, 0.8181818181818182, 10, 0.8333333333333334] + - ["CH", 1442041200000, 198, 393, 1442037600000, 1442073600000, -54, 360, 3, 11, 0.9090909090909091, 11, 0.9166666666666666] + - ["CH", 1442073600000, 360, 753, 1442091600000, null, -54, 360, 3, 12, 1.0, 12, 1.0] + - ["CL", 1442019600000, -370, -370, null, 1442095200000, -370, 390, 1, 1, 0.0, 1, 0.05] + - ["CL", 1442095200000, -276, -646, null, 1442066400000, -370, 390, 1, 2, 0.05263157894736842, 2, 0.1] + - ["CL", 1442066400000, -41, -687, 1442019600000, 1442077200000, -370, 390, 1, 3, 0.10526315789473684, 3, 0.15] + - ["CL", 1442077200000, -15, -702, 1442095200000, 1442059200000, -370, 390, 1, 4, 0.15789473684210525, 4, 0.2] + - ["CL", 1442059200000, -12, -714, 1442066400000, 1442034000000, -370, 390, 1, 5, 0.21052631578947367, 5, 0.25] + - ["CL", 1442034000000, -1, -715, 1442077200000, 1442041200000, -370, 390, 1, 6, 0.2631578947368421, 6, 0.35] + - ["CL", 1442041200000, -1, -716, 1442059200000, 1442037600000, -370, 390, 1, 6, 0.2631578947368421, 6, 0.35] + - ["CL", 1442037600000, 2, -714, 1442034000000, 1442098800000, -370, 390, 2, 8, 0.3684210526315789, 7, 0.4] + - ["CL", 1442098800000, 9, -705, 1442041200000, 1442070000000, -370, 390, 2, 9, 0.42105263157894735, 8, 0.45] + - ["CL", 1442070000000, 13, -692, 1442037600000, 1442023200000, -370, 390, 2, 10, 0.47368421052631576, 9, 0.5] + - ["CL", 1442023200000, 15, -677, 1442098800000, 1442062800000, -370, 390, 2, 11, 0.5263157894736842, 10, 0.55] + - ["CL", 1442062800000, 17, -660, 1442070000000, 1442080800000, -370, 390, 2, 12, 0.5789473684210527, 11, 0.65] + - ["CL", 1442080800000, 17, -643, 1442023200000, 1442091600000, -370, 390, 2, 12, 0.5789473684210527, 11, 0.65] + - ["CL", 1442091600000, 20, -623, 1442062800000, 1442030400000, -370, 390, 2, 14, 0.6842105263157895, 12, 0.7] + - ["CL", 1442030400000, 40, -583, 1442080800000, 1442084400000, -370, 390, 3, 15, 0.7368421052631579, 13, 0.75] + - ["CL", 1442084400000, 126, -457, 1442091600000, 1442073600000, -370, 390, 3, 16, 0.7894736842105263, 14, 0.8] + - ["CL", 1442073600000, 153, -304, 1442030400000, 1442016000000, -370, 390, 3, 17, 0.8421052631578947, 15, 0.85] + - ["CL", 1442016000000, 161, -143, 1442084400000, 1442088000000, -370, 390, 3, 18, 0.8947368421052632, 16, 0.9] + - ["CL", 1442088000000, 286, 143, 1442073600000, 1442052000000, -370, 390, 3, 19, 0.9473684210526315, 17, 0.95] + - ["CL", 1442052000000, 390, 533, 1442016000000, null, -370, 390, 3, 20, 1.0, 18, 1.0] + - ["CN", 1442066400000, -15, -15, null, 1442023200000, -15, 293, 1, 1, 0.0, 1, 0.1] + - ["CN", 1442023200000, -13, -28, null, 1442080800000, -15, 293, 1, 2, 0.1111111111111111, 2, 0.2] + - ["CN", 1442080800000, -10, -38, 1442066400000, 1442084400000, -15, 293, 1, 3, 0.2222222222222222, 3, 0.3] + - ["CN", 1442084400000, -1, -39, 1442023200000, 1442052000000, -15, 293, 1, 4, 0.3333333333333333, 4, 0.4] + - ["CN", 1442052000000, 0, -39, 1442080800000, 1442059200000, -15, 293, 2, 5, 0.4444444444444444, 5, 0.5] + - ["CN", 1442059200000, 8, -31, 1442084400000, 1442055600000, -15, 293, 2, 6, 0.5555555555555556, 6, 0.6] + - ["CN", 1442055600000, 69, 38, 1442052000000, 1442037600000, -15, 293, 2, 7, 0.6666666666666666, 7, 0.7] + - ["CN", 1442037600000, 98, 136, 1442059200000, 1442026800000, -15, 293, 3, 8, 0.7777777777777778, 8, 0.8] + - ["CN", 1442026800000, 154, 290, 1442055600000, 1442048400000, -15, 293, 3, 9, 0.8888888888888888, 9, 0.9] + - ["CN", 1442048400000, 293, 583, 1442037600000, null, -15, 293, 3, 10, 1.0, 10, 1.0] + - ["CO", 1442070000000, -45, -45, null, 1442023200000, -45, 39860, 1, 1, 0.0, 1, 0.06666666666666667] + - ["CO", 1442023200000, 9, -36, null, 1442019600000, -45, 39860, 1, 2, 0.07142857142857142, 2, 0.13333333333333333] + - ["CO", 1442019600000, 12, -24, 1442070000000, 1442016000000, -45, 39860, 1, 3, 0.14285714285714285, 3, 0.2] + - ["CO", 1442016000000, 16, -8, 1442023200000, 1442080800000, -45, 39860, 1, 4, 0.21428571428571427, 4, 0.26666666666666666] + - ["CO", 1442080800000, 25, 17, 1442019600000, 1442084400000, -45, 39860, 1, 5, 0.2857142857142857, 5, 0.3333333333333333] + - ["CO", 1442084400000, 51, 68, 1442016000000, 1442098800000, -45, 39860, 2, 6, 0.35714285714285715, 6, 0.4] + - ["CO", 1442098800000, 83, 151, 1442080800000, 1442066400000, -45, 39860, 2, 7, 0.42857142857142855, 7, 0.4666666666666667] + - ["CO", 1442066400000, 288, 439, 1442084400000, 1442095200000, -45, 39860, 2, 8, 0.5, 8, 0.5333333333333333] + - ["CO", 1442095200000, 290, 729, 1442098800000, 1442091600000, -45, 39860, 2, 9, 0.5714285714285714, 9, 0.6] + - ["CO", 1442091600000, 377, 1106, 1442066400000, 1442030400000, -45, 39860, 2, 10, 0.6428571428571429, 10, 0.6666666666666666] + - ["CO", 1442030400000, 441, 1547, 1442095200000, 1442059200000, -45, 39860, 3, 11, 0.7142857142857143, 11, 0.7333333333333333] + - ["CO", 1442059200000, 473, 2020, 1442091600000, 1442077200000, -45, 39860, 3, 12, 0.7857142857142857, 12, 0.8] + - ["CO", 1442077200000, 581, 2601, 1442030400000, 1442088000000, -45, 39860, 3, 13, 0.8571428571428571, 13, 0.8666666666666667] + - ["CO", 1442088000000, 17150, 19751, 1442059200000, 1442073600000, -45, 39860, 3, 14, 0.9285714285714286, 14, 0.9333333333333333] + - ["CO", 1442073600000, 39860, 59611, 1442077200000, null, -45, 39860, 3, 15, 1.0, 15, 1.0] + - ["CR", 1442041200000, 51, 51, null, 1442019600000, 51, 2497, 1, 1, 0.0, 1, 0.125] + - ["CR", 1442019600000, 62, 113, null, 1442023200000, 51, 2497, 1, 2, 0.14285714285714285, 2, 0.375] + - ["CR", 1442023200000, 62, 175, 1442041200000, 1442088000000, 51, 2497, 1, 2, 0.14285714285714285, 2, 0.375] + - ["CR", 1442088000000, 72, 247, 1442019600000, 1442026800000, 51, 2497, 2, 4, 0.42857142857142855, 3, 0.5] + - ["CR", 1442026800000, 140, 387, 1442023200000, 1442048400000, 51, 2497, 2, 5, 0.5714285714285714, 4, 0.625] + - ["CR", 1442048400000, 163, 550, 1442088000000, 1442044800000, 51, 2497, 2, 6, 0.7142857142857143, 5, 0.75] + - ["CR", 1442044800000, 194, 744, 1442026800000, 1442030400000, 51, 2497, 3, 7, 0.8571428571428571, 6, 0.875] + - ["CR", 1442030400000, 2497, 3241, 1442048400000, null, 51, 2497, 3, 8, 1.0, 7, 1.0] + - ["CZ", 1442080800000, -28, -28, null, 1442026800000, -28, 2051, 1, 1, 0.0, 1, 0.09090909090909091] + - ["CZ", 1442026800000, -19, -47, null, 1442062800000, -28, 2051, 1, 2, 0.1, 2, 0.18181818181818182] + - ["CZ", 1442062800000, 0, -47, 1442080800000, 1442098800000, -28, 2051, 1, 3, 0.2, 3, 0.2727272727272727] + - ["CZ", 1442098800000, 2, -45, 1442026800000, 1442037600000, -28, 2051, 1, 4, 0.3, 4, 0.36363636363636365] + - ["CZ", 1442037600000, 18, -27, 1442062800000, 1442059200000, -28, 2051, 2, 5, 0.4, 5, 0.45454545454545453] + - ["CZ", 1442059200000, 21, -6, 1442098800000, 1442034000000, -28, 2051, 2, 6, 0.5, 6, 0.5454545454545454] + - ["CZ", 1442034000000, 78, 72, 1442037600000, 1442077200000, -28, 2051, 2, 7, 0.6, 7, 0.6363636363636364] + - ["CZ", 1442077200000, 115, 187, 1442059200000, 1442070000000, -28, 2051, 2, 8, 0.7, 8, 0.7272727272727273] + - ["CZ", 1442070000000, 168, 355, 1442034000000, 1442055600000, -28, 2051, 3, 9, 0.8, 9, 0.8181818181818182] + - ["CZ", 1442055600000, 1073, 1428, 1442077200000, 1442073600000, -28, 2051, 3, 10, 0.9, 10, 0.9090909090909091] + - ["CZ", 1442073600000, 2051, 3479, 1442070000000, null, -28, 2051, 3, 11, 1.0, 11, 1.0] + - ["DE", 1442084400000, -125, -125, null, 1442019600000, -125, 6075, 1, 1, 0.0, 1, 0.043478260869565216] + - ["DE", 1442019600000, 0, -125, null, 1442023200000, -125, 6075, 1, 2, 0.045454545454545456, 2, 0.08695652173913043] + - ["DE", 1442023200000, 64, -61, 1442084400000, 1442016000000, -125, 6075, 1, 3, 0.09090909090909091, 3, 0.13043478260869565] + - ["DE", 1442016000000, 167, 106, 1442019600000, 1442088000000, -125, 6075, 1, 4, 0.13636363636363635, 4, 0.17391304347826086] + - ["DE", 1442088000000, 190, 296, 1442023200000, 1442041200000, -125, 6075, 1, 5, 0.18181818181818182, 5, 0.21739130434782608] + - ["DE", 1442041200000, 197, 493, 1442016000000, 1442062800000, -125, 6075, 1, 6, 0.22727272727272727, 6, 0.2608695652173913] + - ["DE", 1442062800000, 283, 776, 1442088000000, 1442059200000, -125, 6075, 1, 7, 0.2727272727272727, 7, 0.30434782608695654] + - ["DE", 1442059200000, 289, 1065, 1442041200000, 1442098800000, -125, 6075, 1, 8, 0.3181818181818182, 8, 0.34782608695652173] + - ["DE", 1442098800000, 329, 1394, 1442062800000, 1442034000000, -125, 6075, 2, 9, 0.36363636363636365, 9, 0.391304347826087] + - ["DE", 1442034000000, 358, 1752, 1442059200000, 1442030400000, -125, 6075, 2, 10, 0.4090909090909091, 10, 0.43478260869565216] + - ["DE", 1442030400000, 373, 2125, 1442098800000, 1442037600000, -125, 6075, 2, 11, 0.45454545454545453, 11, 0.4782608695652174] + - ["DE", 1442037600000, 544, 2669, 1442034000000, 1442048400000, -125, 6075, 2, 12, 0.5, 12, 0.5217391304347826] + - ["DE", 1442048400000, 811, 3480, 1442030400000, 1442044800000, -125, 6075, 2, 13, 0.5454545454545454, 13, 0.5652173913043478] + - ["DE", 1442044800000, 979, 4459, 1442037600000, 1442095200000, -125, 6075, 2, 14, 0.5909090909090909, 14, 0.6086956521739131] + - ["DE", 1442095200000, 1007, 5466, 1442048400000, 1442080800000, -125, 6075, 2, 15, 0.6363636363636364, 15, 0.6521739130434783] + - ["DE", 1442080800000, 1133, 6599, 1442044800000, 1442055600000, -125, 6075, 2, 16, 0.6818181818181818, 16, 0.6956521739130435] + - ["DE", 1442055600000, 1523, 8122, 1442095200000, 1442066400000, -125, 6075, 3, 17, 0.7272727272727273, 17, 0.7391304347826086] + - ["DE", 1442066400000, 1577, 9699, 1442080800000, 1442052000000, -125, 6075, 3, 18, 0.7727272727272727, 18, 0.782608695652174] + - ["DE", 1442052000000, 1600, 11299, 1442055600000, 1442070000000, -125, 6075, 3, 19, 0.8181818181818182, 19, 0.8260869565217391] + - ["DE", 1442070000000, 1666, 12965, 1442066400000, 1442077200000, -125, 6075, 3, 20, 0.8636363636363636, 20, 0.8695652173913043] + - ["DE", 1442077200000, 2188, 15153, 1442052000000, 1442091600000, -125, 6075, 3, 21, 0.9090909090909091, 21, 0.9130434782608695] + - ["DE", 1442091600000, 4355, 19508, 1442070000000, 1442073600000, -125, 6075, 3, 22, 0.9545454545454546, 22, 0.9565217391304348] + - ["DE", 1442073600000, 6075, 25583, 1442077200000, null, -125, 6075, 3, 23, 1.0, 23, 1.0] + - ["DK", 1442084400000, -97, -97, null, 1442077200000, -97, 416, 1, 1, 0.0, 1, 0.08333333333333333] + - ["DK", 1442077200000, -9, -106, null, 1442048400000, -97, 416, 1, 2, 0.09090909090909091, 2, 0.16666666666666666] + - ["DK", 1442048400000, -5, -111, 1442084400000, 1442059200000, -97, 416, 1, 3, 0.18181818181818182, 3, 0.25] + - ["DK", 1442059200000, 0, -111, 1442077200000, 1442095200000, -97, 416, 1, 4, 0.2727272727272727, 4, 0.4166666666666667] + - ["DK", 1442095200000, 0, -111, 1442048400000, 1442062800000, -97, 416, 2, 4, 0.2727272727272727, 4, 0.4166666666666667] + - ["DK", 1442062800000, 1, -110, 1442059200000, 1442037600000, -97, 416, 2, 6, 0.45454545454545453, 5, 0.5] + - ["DK", 1442037600000, 10, -100, 1442095200000, 1442044800000, -97, 416, 2, 7, 0.5454545454545454, 6, 0.5833333333333334] + - ["DK", 1442044800000, 36, -64, 1442062800000, 1442055600000, -97, 416, 2, 8, 0.6363636363636364, 7, 0.6666666666666666] + - ["DK", 1442055600000, 42, -22, 1442037600000, 1442080800000, -97, 416, 3, 9, 0.7272727272727273, 8, 0.75] + - ["DK", 1442080800000, 61, 39, 1442044800000, 1442091600000, -97, 416, 3, 10, 0.8181818181818182, 9, 0.8333333333333334] + - ["DK", 1442091600000, 139, 178, 1442055600000, 1442066400000, -97, 416, 3, 11, 0.9090909090909091, 10, 0.9166666666666666] + - ["DK", 1442066400000, 416, 594, 1442080800000, null, -97, 416, 3, 12, 1.0, 11, 1.0] + - ["DO", 1442023200000, 8, 8, null, 1442084400000, 8, 200, 1, 1, 0.0, 1, 0.4] + - ["DO", 1442084400000, 8, 16, null, 1442095200000, 8, 200, 1, 1, 0.0, 1, 0.4] + - ["DO", 1442095200000, 13, 29, 1442023200000, 1442066400000, 8, 200, 2, 3, 0.5, 2, 0.6] + - ["DO", 1442066400000, 35, 64, 1442084400000, 1442073600000, 8, 200, 2, 4, 0.75, 3, 0.8] + - ["DO", 1442073600000, 200, 264, 1442095200000, null, 8, 200, 3, 5, 1.0, 4, 1.0] + - ["DZ", 1442077200000, -1, -1, null, null, -1, -1, 1, 1, 0.0, 1, 1.0] + - ["EC", 1442077200000, -366, -366, null, 1442023200000, -366, 568, 1, 1, 0.0, 1, 0.16666666666666666] + - ["EC", 1442023200000, -9, -375, null, 1442030400000, -366, 568, 1, 2, 0.2, 2, 0.3333333333333333] + - ["EC", 1442030400000, 0, -375, 1442077200000, 1442095200000, -366, 568, 2, 3, 0.4, 3, 0.5] + - ["EC", 1442095200000, 10, -365, 1442023200000, 1442019600000, -366, 568, 2, 4, 0.6, 4, 0.6666666666666666] + - ["EC", 1442019600000, 29, -336, 1442030400000, 1442084400000, -366, 568, 3, 5, 0.8, 5, 0.8333333333333334] + - ["EC", 1442084400000, 568, 232, 1442095200000, null, -366, 568, 3, 6, 1.0, 6, 1.0] + - ["EE", 1442044800000, -19, -19, null, 1442041200000, -19, 37, 1, 1, 0.0, 1, 0.5] + - ["EE", 1442041200000, 37, 18, null, null, -19, 37, 2, 2, 1.0, 2, 1.0] + - ["EG", 1442073600000, 1, 1, null, 1442055600000, 1, 112, 1, 1, 0.0, 1, 0.2] + - ["EG", 1442055600000, 14, 15, null, 1442026800000, 1, 112, 1, 2, 0.25, 2, 0.4] + - ["EG", 1442026800000, 16, 31, 1442073600000, 1442091600000, 1, 112, 2, 3, 0.5, 3, 0.6] + - ["EG", 1442091600000, 27, 58, 1442055600000, 1442062800000, 1, 112, 2, 4, 0.75, 4, 0.8] + - ["EG", 1442062800000, 112, 170, 1442026800000, null, 1, 112, 3, 5, 1.0, 5, 1.0] + - ["ES", 1442044800000, -169, -169, null, 1442088000000, -169, 2506, 1, 1, 0.0, 1, 0.05] + - ["ES", 1442088000000, -130, -299, null, 1442062800000, -169, 2506, 1, 2, 0.05263157894736842, 2, 0.1] + - ["ES", 1442062800000, -71, -370, 1442044800000, 1442034000000, -169, 2506, 1, 3, 0.10526315789473684, 3, 0.15] + - ["ES", 1442034000000, -52, -422, 1442088000000, 1442023200000, -169, 2506, 1, 4, 0.15789473684210525, 4, 0.2] + - ["ES", 1442023200000, -5, -427, 1442062800000, 1442052000000, -169, 2506, 1, 5, 0.21052631578947367, 5, 0.25] + - ["ES", 1442052000000, -4, -431, 1442034000000, 1442037600000, -169, 2506, 1, 6, 0.2631578947368421, 6, 0.3] + - ["ES", 1442037600000, 3, -428, 1442023200000, 1442070000000, -169, 2506, 1, 7, 0.3157894736842105, 7, 0.35] + - ["ES", 1442070000000, 61, -367, 1442052000000, 1442019600000, -169, 2506, 2, 8, 0.3684210526315789, 8, 0.4] + - ["ES", 1442019600000, 103, -264, 1442037600000, 1442041200000, -169, 2506, 2, 9, 0.42105263157894735, 9, 0.45] + - ["ES", 1442041200000, 118, -146, 1442070000000, 1442073600000, -169, 2506, 2, 10, 0.47368421052631576, 10, 0.5] + - ["ES", 1442073600000, 154, 8, 1442019600000, 1442048400000, -169, 2506, 2, 11, 0.5263157894736842, 11, 0.55] + - ["ES", 1442048400000, 158, 166, 1442041200000, 1442084400000, -169, 2506, 2, 12, 0.5789473684210527, 12, 0.6] + - ["ES", 1442084400000, 337, 503, 1442073600000, 1442098800000, -169, 2506, 2, 13, 0.631578947368421, 13, 0.65] + - ["ES", 1442098800000, 458, 961, 1442048400000, 1442066400000, -169, 2506, 2, 14, 0.6842105263157895, 14, 0.7] + - ["ES", 1442066400000, 461, 1422, 1442084400000, 1442055600000, -169, 2506, 3, 15, 0.7368421052631579, 15, 0.75] + - ["ES", 1442055600000, 495, 1917, 1442098800000, 1442091600000, -169, 2506, 3, 16, 0.7894736842105263, 16, 0.8] + - ["ES", 1442091600000, 700, 2617, 1442066400000, 1442059200000, -169, 2506, 3, 17, 0.8421052631578947, 17, 0.85] + - ["ES", 1442059200000, 1086, 3703, 1442055600000, 1442077200000, -169, 2506, 3, 18, 0.8947368421052632, 18, 0.9] + - ["ES", 1442077200000, 1240, 4943, 1442091600000, 1442095200000, -169, 2506, 3, 19, 0.9473684210526315, 19, 0.95] + - ["ES", 1442095200000, 2506, 7449, 1442059200000, null, -169, 2506, 3, 20, 1.0, 20, 1.0] + - ["FI", 1442073600000, -1, -1, null, 1442048400000, -1, 1491, 1, 1, 0.0, 1, 0.08333333333333333] + - ["FI", 1442048400000, 12, 11, null, 1442037600000, -1, 1491, 1, 2, 0.09090909090909091, 2, 0.16666666666666666] + - ["FI", 1442037600000, 14, 25, 1442073600000, 1442062800000, -1, 1491, 1, 3, 0.18181818181818182, 3, 0.25] + - ["FI", 1442062800000, 19, 44, 1442048400000, 1442095200000, -1, 1491, 1, 4, 0.2727272727272727, 4, 0.3333333333333333] + - ["FI", 1442095200000, 69, 113, 1442037600000, 1442080800000, -1, 1491, 2, 5, 0.36363636363636365, 5, 0.4166666666666667] + - ["FI", 1442080800000, 104, 217, 1442062800000, 1442066400000, -1, 1491, 2, 6, 0.45454545454545453, 6, 0.5] + - ["FI", 1442066400000, 183, 400, 1442095200000, 1442052000000, -1, 1491, 2, 7, 0.5454545454545454, 7, 0.5833333333333334] + - ["FI", 1442052000000, 186, 586, 1442080800000, 1442077200000, -1, 1491, 2, 8, 0.6363636363636364, 8, 0.6666666666666666] + - ["FI", 1442077200000, 200, 786, 1442066400000, 1442059200000, -1, 1491, 3, 9, 0.7272727272727273, 9, 0.75] + - ["FI", 1442059200000, 407, 1193, 1442052000000, 1442084400000, -1, 1491, 3, 10, 0.8181818181818182, 10, 0.8333333333333334] + - ["FI", 1442084400000, 895, 2088, 1442077200000, 1442030400000, -1, 1491, 3, 11, 0.9090909090909091, 11, 0.9166666666666666] + - ["FI", 1442030400000, 1491, 3579, 1442059200000, null, -1, 1491, 3, 12, 1.0, 12, 1.0] + - ["FR", 1442077200000, -444, -444, null, 1442016000000, -444, 6643, 1, 1, 0.0, 1, 0.043478260869565216] + - ["FR", 1442016000000, -1, -445, null, 1442026800000, -444, 6643, 1, 2, 0.045454545454545456, 2, 0.08695652173913043] + - ["FR", 1442026800000, 86, -359, 1442077200000, 1442095200000, -444, 6643, 1, 3, 0.09090909090909091, 3, 0.13043478260869565] + - ["FR", 1442095200000, 87, -272, 1442016000000, 1442098800000, -444, 6643, 1, 4, 0.13636363636363635, 4, 0.17391304347826086] + - ["FR", 1442098800000, 136, -136, 1442026800000, 1442044800000, -444, 6643, 1, 5, 0.18181818181818182, 5, 0.21739130434782608] + - ["FR", 1442044800000, 172, 36, 1442095200000, 1442055600000, -444, 6643, 1, 6, 0.22727272727272727, 6, 0.2608695652173913] + - ["FR", 1442055600000, 463, 499, 1442098800000, 1442070000000, -444, 6643, 1, 7, 0.2727272727272727, 7, 0.30434782608695654] + - ["FR", 1442070000000, 474, 973, 1442044800000, 1442034000000, -444, 6643, 1, 8, 0.3181818181818182, 8, 0.34782608695652173] + - ["FR", 1442034000000, 476, 1449, 1442055600000, 1442080800000, -444, 6643, 2, 9, 0.36363636363636365, 9, 0.391304347826087] + - ["FR", 1442080800000, 557, 2006, 1442070000000, 1442019600000, -444, 6643, 2, 10, 0.4090909090909091, 10, 0.43478260869565216] + - ["FR", 1442019600000, 585, 2591, 1442034000000, 1442041200000, -444, 6643, 2, 11, 0.45454545454545453, 11, 0.4782608695652174] + - ["FR", 1442041200000, 604, 3195, 1442080800000, 1442023200000, -444, 6643, 2, 12, 0.5, 12, 0.5217391304347826] + - ["FR", 1442023200000, 628, 3823, 1442019600000, 1442052000000, -444, 6643, 2, 13, 0.5454545454545454, 13, 0.5652173913043478] + - ["FR", 1442052000000, 637, 4460, 1442041200000, 1442091600000, -444, 6643, 2, 14, 0.5909090909090909, 14, 0.6086956521739131] + - ["FR", 1442091600000, 741, 5201, 1442023200000, 1442088000000, -444, 6643, 2, 15, 0.6363636363636364, 15, 0.6521739130434783] + - ["FR", 1442088000000, 1872, 7073, 1442052000000, 1442066400000, -444, 6643, 2, 16, 0.6818181818181818, 16, 0.6956521739130435] + - ["FR", 1442066400000, 2516, 9589, 1442091600000, 1442048400000, -444, 6643, 3, 17, 0.7272727272727273, 17, 0.7391304347826086] + - ["FR", 1442048400000, 3027, 12616, 1442088000000, 1442073600000, -444, 6643, 3, 18, 0.7727272727272727, 18, 0.782608695652174] + - ["FR", 1442073600000, 3522, 16138, 1442066400000, 1442037600000, -444, 6643, 3, 19, 0.8181818181818182, 19, 0.8260869565217391] + - ["FR", 1442037600000, 4174, 20312, 1442048400000, 1442059200000, -444, 6643, 3, 20, 0.8636363636363636, 20, 0.8695652173913043] + - ["FR", 1442059200000, 4650, 24962, 1442073600000, 1442062800000, -444, 6643, 3, 21, 0.9090909090909091, 21, 0.9130434782608695] + - ["FR", 1442062800000, 5676, 30638, 1442037600000, 1442084400000, -444, 6643, 3, 22, 0.9545454545454546, 22, 0.9565217391304348] + - ["FR", 1442084400000, 6643, 37281, 1442059200000, null, -444, 6643, 3, 23, 1.0, 23, 1.0] + - ["GB", 1442016000000, -44, -44, null, 1442034000000, -44, 16111, 1, 1, 0.0, 1, 0.041666666666666664] + - ["GB", 1442034000000, -12, -56, null, 1442044800000, -44, 16111, 1, 2, 0.043478260869565216, 2, 0.08333333333333333] + - ["GB", 1442044800000, 32, -24, 1442016000000, 1442041200000, -44, 16111, 1, 3, 0.08695652173913043, 3, 0.125] + - ["GB", 1442041200000, 42, 18, 1442034000000, 1442098800000, -44, 16111, 1, 4, 0.13043478260869565, 4, 0.16666666666666666] + - ["GB", 1442098800000, 49, 67, 1442044800000, 1442019600000, -44, 16111, 1, 5, 0.17391304347826086, 5, 0.20833333333333334] + - ["GB", 1442019600000, 54, 121, 1442041200000, 1442052000000, -44, 16111, 1, 6, 0.21739130434782608, 6, 0.25] + - ["GB", 1442052000000, 168, 289, 1442098800000, 1442095200000, -44, 16111, 1, 7, 0.2608695652173913, 7, 0.2916666666666667] + - ["GB", 1442095200000, 238, 527, 1442019600000, 1442026800000, -44, 16111, 1, 8, 0.30434782608695654, 8, 0.3333333333333333] + - ["GB", 1442026800000, 339, 866, 1442052000000, 1442070000000, -44, 16111, 2, 9, 0.34782608695652173, 9, 0.375] + - ["GB", 1442070000000, 374, 1240, 1442095200000, 1442084400000, -44, 16111, 2, 10, 0.391304347826087, 10, 0.4166666666666667] + - ["GB", 1442084400000, 384, 1624, 1442026800000, 1442055600000, -44, 16111, 2, 11, 0.43478260869565216, 11, 0.4583333333333333] + - ["GB", 1442055600000, 453, 2077, 1442070000000, 1442037600000, -44, 16111, 2, 12, 0.4782608695652174, 12, 0.5] + - ["GB", 1442037600000, 544, 2621, 1442084400000, 1442073600000, -44, 16111, 2, 13, 0.5217391304347826, 13, 0.5416666666666666] + - ["GB", 1442073600000, 648, 3269, 1442055600000, 1442066400000, -44, 16111, 2, 14, 0.5652173913043478, 14, 0.5833333333333334] + - ["GB", 1442066400000, 671, 3940, 1442037600000, 1442048400000, -44, 16111, 2, 15, 0.6086956521739131, 15, 0.625] + - ["GB", 1442048400000, 740, 4680, 1442073600000, 1442091600000, -44, 16111, 2, 16, 0.6521739130434783, 16, 0.6666666666666666] + - ["GB", 1442091600000, 811, 5491, 1442066400000, 1442077200000, -44, 16111, 3, 17, 0.6956521739130435, 17, 0.7083333333333334] + - ["GB", 1442077200000, 1135, 6626, 1442048400000, 1442080800000, -44, 16111, 3, 18, 0.7391304347826086, 18, 0.75] + - ["GB", 1442080800000, 1444, 8070, 1442091600000, 1442088000000, -44, 16111, 3, 19, 0.782608695652174, 19, 0.7916666666666666] + - ["GB", 1442088000000, 1593, 9663, 1442077200000, 1442023200000, -44, 16111, 3, 20, 0.8260869565217391, 20, 0.8333333333333334] + - ["GB", 1442023200000, 1816, 11479, 1442080800000, 1442030400000, -44, 16111, 3, 21, 0.8695652173913043, 21, 0.875] + - ["GB", 1442030400000, 2524, 14003, 1442088000000, 1442062800000, -44, 16111, 3, 22, 0.9130434782608695, 22, 0.9166666666666666] + - ["GB", 1442062800000, 5743, 19746, 1442023200000, 1442059200000, -44, 16111, 3, 23, 0.9565217391304348, 23, 0.9583333333333334] + - ["GB", 1442059200000, 16111, 35857, 1442030400000, null, -44, 16111, 3, 24, 1.0, 24, 1.0] + - ["GE", 1442052000000, -108, -108, null, 1442080800000, -108, 16, 1, 1, 0.0, 1, 0.25] + - ["GE", 1442080800000, -27, -135, null, 1442044800000, -108, 16, 1, 2, 0.3333333333333333, 2, 0.5] + - ["GE", 1442044800000, -21, -156, 1442052000000, 1442062800000, -108, 16, 2, 3, 0.6666666666666666, 3, 0.75] + - ["GE", 1442062800000, 16, -140, 1442080800000, null, -108, 16, 3, 4, 1.0, 4, 1.0] + - ["GH", 1442088000000, 0, 0, null, null, 0, 0, 1, 1, 0.0, 1, 1.0] + - ["GR", 1442073600000, -314, -314, null, 1442048400000, -314, 179, 1, 1, 0.0, 1, 0.1] + - ["GR", 1442048400000, -26, -340, null, 1442034000000, -314, 179, 1, 2, 0.1111111111111111, 2, 0.2] + - ["GR", 1442034000000, 0, -340, 1442073600000, 1442070000000, -314, 179, 1, 3, 0.2222222222222222, 3, 0.3] + - ["GR", 1442070000000, 2, -338, 1442048400000, 1442041200000, -314, 179, 1, 4, 0.3333333333333333, 4, 0.4] + - ["GR", 1442041200000, 7, -331, 1442034000000, 1442062800000, -314, 179, 2, 5, 0.4444444444444444, 5, 0.5] + - ["GR", 1442062800000, 8, -323, 1442070000000, 1442019600000, -314, 179, 2, 6, 0.5555555555555556, 6, 0.6] + - ["GR", 1442019600000, 82, -241, 1442041200000, 1442080800000, -314, 179, 2, 7, 0.6666666666666666, 7, 0.7] + - ["GR", 1442080800000, 88, -153, 1442062800000, 1442091600000, -314, 179, 3, 8, 0.7777777777777778, 8, 0.8] + - ["GR", 1442091600000, 123, -30, 1442019600000, 1442084400000, -314, 179, 3, 9, 0.8888888888888888, 9, 0.9] + - ["GR", 1442084400000, 179, 149, 1442080800000, null, -314, 179, 3, 10, 1.0, 10, 1.0] + - ["GT", 1442023200000, -167, -167, null, 1442098800000, -167, 173, 1, 1, 0.0, 1, 0.3333333333333333] + - ["GT", 1442098800000, 1, -166, null, 1442026800000, -167, 173, 2, 2, 0.5, 2, 0.6666666666666666] + - ["GT", 1442026800000, 173, 7, 1442023200000, null, -167, 173, 3, 3, 1.0, 3, 1.0] + - ["HK", 1442026800000, -211, -211, null, 1442019600000, -211, 5545, 1, 1, 0.0, 1, 0.05263157894736842] + - ["HK", 1442019600000, -113, -324, null, 1442041200000, -211, 5545, 1, 2, 0.05555555555555555, 2, 0.10526315789473684] + - ["HK", 1442041200000, -15, -339, 1442026800000, 1442091600000, -211, 5545, 1, 3, 0.1111111111111111, 3, 0.15789473684210525] + - ["HK", 1442091600000, -3, -342, 1442019600000, 1442095200000, -211, 5545, 1, 4, 0.16666666666666666, 4, 0.21052631578947367] + - ["HK", 1442095200000, -1, -343, 1442041200000, 1442080800000, -211, 5545, 1, 5, 0.2222222222222222, 5, 0.2631578947368421] + - ["HK", 1442080800000, 0, -343, 1442091600000, 1442048400000, -211, 5545, 1, 6, 0.2777777777777778, 6, 0.3157894736842105] + - ["HK", 1442048400000, 1, -342, 1442095200000, 1442062800000, -211, 5545, 1, 7, 0.3333333333333333, 7, 0.42105263157894735] + - ["HK", 1442062800000, 1, -341, 1442080800000, 1442059200000, -211, 5545, 2, 7, 0.3333333333333333, 7, 0.42105263157894735] + - ["HK", 1442059200000, 2, -339, 1442048400000, 1442052000000, -211, 5545, 2, 9, 0.4444444444444444, 8, 0.47368421052631576] + - ["HK", 1442052000000, 15, -324, 1442062800000, 1442044800000, -211, 5545, 2, 10, 0.5, 9, 0.5263157894736842] + - ["HK", 1442044800000, 21, -303, 1442059200000, 1442066400000, -211, 5545, 2, 11, 0.5555555555555556, 10, 0.5789473684210527] + - ["HK", 1442066400000, 39, -264, 1442052000000, 1442030400000, -211, 5545, 2, 12, 0.6111111111111112, 11, 0.631578947368421] + - ["HK", 1442030400000, 157, -107, 1442044800000, 1442070000000, -211, 5545, 2, 13, 0.6666666666666666, 12, 0.6842105263157895] + - ["HK", 1442070000000, 314, 207, 1442066400000, 1442037600000, -211, 5545, 3, 14, 0.7222222222222222, 13, 0.7368421052631579] + - ["HK", 1442037600000, 636, 843, 1442030400000, 1442055600000, -211, 5545, 3, 15, 0.7777777777777778, 14, 0.7894736842105263] + - ["HK", 1442055600000, 804, 1647, 1442070000000, 1442034000000, -211, 5545, 3, 16, 0.8333333333333334, 15, 0.8421052631578947] + - ["HK", 1442034000000, 1137, 2784, 1442037600000, 1442023200000, -211, 5545, 3, 17, 0.8888888888888888, 16, 0.8947368421052632] + - ["HK", 1442023200000, 2414, 5198, 1442055600000, 1442073600000, -211, 5545, 3, 18, 0.9444444444444444, 17, 0.9473684210526315] + - ["HK", 1442073600000, 5545, 10743, 1442034000000, null, -211, 5545, 3, 19, 1.0, 18, 1.0] + - ["HN", 1442026800000, -1, -1, null, null, -1, -1, 1, 1, 0.0, 1, 1.0] + - ["HR", 1442084400000, -10, -10, null, 1442073600000, -10, 220, 1, 1, 0.0, 1, 0.16666666666666666] + - ["HR", 1442073600000, 0, -10, null, 1442070000000, -10, 220, 1, 2, 0.2, 2, 0.3333333333333333] + - ["HR", 1442070000000, 32, 22, 1442084400000, 1442077200000, -10, 220, 2, 3, 0.4, 3, 0.5] + - ["HR", 1442077200000, 58, 80, 1442073600000, 1442088000000, -10, 220, 2, 4, 0.6, 4, 0.6666666666666666] + - ["HR", 1442088000000, 82, 162, 1442070000000, 1442080800000, -10, 220, 3, 5, 0.8, 5, 0.8333333333333334] + - ["HR", 1442080800000, 220, 382, 1442077200000, null, -10, 220, 3, 6, 1.0, 6, 1.0] + - ["HU", 1442088000000, -71, -71, null, 1442091600000, -71, 547, 1, 1, 0.0, 1, 0.07692307692307693] + - ["HU", 1442091600000, -5, -76, null, 1442055600000, -71, 547, 1, 2, 0.08333333333333333, 2, 0.15384615384615385] + - ["HU", 1442055600000, -2, -78, 1442088000000, 1442019600000, -71, 547, 1, 3, 0.16666666666666666, 3, 0.23076923076923078] + - ["HU", 1442019600000, 46, -32, 1442091600000, 1442062800000, -71, 547, 1, 4, 0.25, 4, 0.3076923076923077] + - ["HU", 1442062800000, 50, 18, 1442055600000, 1442041200000, -71, 547, 1, 5, 0.3333333333333333, 5, 0.38461538461538464] + - ["HU", 1442041200000, 91, 109, 1442019600000, 1442098800000, -71, 547, 2, 6, 0.4166666666666667, 6, 0.46153846153846156] + - ["HU", 1442098800000, 110, 219, 1442062800000, 1442084400000, -71, 547, 2, 7, 0.5, 7, 0.5384615384615384] + - ["HU", 1442084400000, 141, 360, 1442041200000, 1442037600000, -71, 547, 2, 8, 0.5833333333333334, 8, 0.6153846153846154] + - ["HU", 1442037600000, 197, 557, 1442098800000, 1442080800000, -71, 547, 2, 9, 0.6666666666666666, 9, 0.6923076923076923] + - ["HU", 1442080800000, 242, 799, 1442084400000, 1442095200000, -71, 547, 3, 10, 0.75, 10, 0.7692307692307693] + - ["HU", 1442095200000, 271, 1070, 1442037600000, 1442048400000, -71, 547, 3, 11, 0.8333333333333334, 11, 0.8461538461538461] + - ["HU", 1442048400000, 499, 1569, 1442080800000, 1442044800000, -71, 547, 3, 12, 0.9166666666666666, 12, 0.9230769230769231] + - ["HU", 1442044800000, 547, 2116, 1442095200000, null, -71, 547, 3, 13, 1.0, 13, 1.0] + - ["ID", 1442026800000, -416, -416, null, 1442044800000, -416, 279, 1, 1, 0.0, 1, 0.07692307692307693] + - ["ID", 1442044800000, -388, -804, null, 1442041200000, -416, 279, 1, 2, 0.08333333333333333, 2, 0.15384615384615385] + - ["ID", 1442041200000, 2, -802, 1442026800000, 1442098800000, -416, 279, 1, 3, 0.16666666666666666, 3, 0.23076923076923078] + - ["ID", 1442098800000, 13, -789, 1442044800000, 1442037600000, -416, 279, 1, 4, 0.25, 4, 0.3076923076923077] + - ["ID", 1442037600000, 14, -775, 1442041200000, 1442055600000, -416, 279, 1, 5, 0.3333333333333333, 5, 0.38461538461538464] + - ["ID", 1442055600000, 16, -759, 1442098800000, 1442059200000, -416, 279, 2, 6, 0.4166666666666667, 6, 0.46153846153846156] + - ["ID", 1442059200000, 17, -742, 1442037600000, 1442034000000, -416, 279, 2, 7, 0.5, 7, 0.5384615384615384] + - ["ID", 1442034000000, 19, -723, 1442055600000, 1442095200000, -416, 279, 2, 8, 0.5833333333333334, 8, 0.6153846153846154] + - ["ID", 1442095200000, 20, -703, 1442059200000, 1442091600000, -416, 279, 2, 9, 0.6666666666666666, 9, 0.6923076923076923] + - ["ID", 1442091600000, 21, -682, 1442034000000, 1442070000000, -416, 279, 3, 10, 0.75, 10, 0.7692307692307693] + - ["ID", 1442070000000, 42, -640, 1442095200000, 1442023200000, -416, 279, 3, 11, 0.8333333333333334, 11, 0.8461538461538461] + - ["ID", 1442023200000, 106, -534, 1442091600000, 1442030400000, -416, 279, 3, 12, 0.9166666666666666, 12, 0.9230769230769231] + - ["ID", 1442030400000, 279, -255, 1442070000000, null, -416, 279, 3, 13, 1.0, 13, 1.0] + - ["IE", 1442070000000, -100, -100, null, 1442091600000, -100, 1062, 1, 1, 0.0, 1, 0.125] + - ["IE", 1442091600000, -71, -171, null, 1442026800000, -100, 1062, 1, 2, 0.14285714285714285, 2, 0.25] + - ["IE", 1442026800000, 1, -170, 1442070000000, 1442030400000, -100, 1062, 1, 3, 0.2857142857142857, 3, 0.5] + - ["IE", 1442030400000, 1, -169, 1442091600000, 1442048400000, -100, 1062, 2, 3, 0.2857142857142857, 3, 0.5] + - ["IE", 1442048400000, 27, -142, 1442026800000, 1442077200000, -100, 1062, 2, 5, 0.5714285714285714, 4, 0.625] + - ["IE", 1442077200000, 403, 261, 1442030400000, 1442084400000, -100, 1062, 2, 6, 0.7142857142857143, 5, 0.75] + - ["IE", 1442084400000, 819, 1080, 1442048400000, 1442066400000, -100, 1062, 3, 7, 0.8571428571428571, 6, 0.875] + - ["IE", 1442066400000, 1062, 2142, 1442077200000, null, -100, 1062, 3, 8, 1.0, 7, 1.0] + - ["IL", 1442095200000, 0, 0, null, 1442066400000, 0, 2745, 1, 1, 0.0, 1, 0.0625] + - ["IL", 1442066400000, 3, 3, null, 1442098800000, 0, 2745, 1, 2, 0.06666666666666667, 2, 0.1875] + - ["IL", 1442098800000, 3, 6, 1442095200000, 1442055600000, 0, 2745, 1, 2, 0.06666666666666667, 2, 0.1875] + - ["IL", 1442055600000, 4, 10, 1442066400000, 1442048400000, 0, 2745, 1, 4, 0.2, 3, 0.25] + - ["IL", 1442048400000, 25, 35, 1442098800000, 1442073600000, 0, 2745, 1, 5, 0.26666666666666666, 4, 0.3125] + - ["IL", 1442073600000, 31, 66, 1442055600000, 1442041200000, 0, 2745, 1, 6, 0.3333333333333333, 5, 0.375] + - ["IL", 1442041200000, 35, 101, 1442048400000, 1442070000000, 0, 2745, 2, 7, 0.4, 6, 0.4375] + - ["IL", 1442070000000, 49, 150, 1442073600000, 1442080800000, 0, 2745, 2, 8, 0.4666666666666667, 7, 0.5] + - ["IL", 1442080800000, 88, 238, 1442041200000, 1442062800000, 0, 2745, 2, 9, 0.5333333333333333, 8, 0.5625] + - ["IL", 1442062800000, 180, 418, 1442070000000, 1442077200000, 0, 2745, 2, 10, 0.6, 9, 0.625] + - ["IL", 1442077200000, 187, 605, 1442080800000, 1442044800000, 0, 2745, 2, 11, 0.6666666666666666, 10, 0.6875] + - ["IL", 1442044800000, 218, 823, 1442062800000, 1442091600000, 0, 2745, 3, 12, 0.7333333333333333, 11, 0.75] + - ["IL", 1442091600000, 707, 1530, 1442077200000, 1442084400000, 0, 2745, 3, 13, 0.8, 12, 0.8125] + - ["IL", 1442084400000, 1137, 2667, 1442044800000, 1442059200000, 0, 2745, 3, 14, 0.8666666666666667, 13, 0.875] + - ["IL", 1442059200000, 1205, 3872, 1442091600000, 1442052000000, 0, 2745, 3, 15, 0.9333333333333333, 14, 0.9375] + - ["IL", 1442052000000, 2745, 6617, 1442084400000, null, 0, 2745, 3, 16, 1.0, 15, 1.0] + - ["IN", 1442023200000, -142, -142, null, 1442080800000, -142, 12091, 1, 1, 0.0, 1, 0.045454545454545456] + - ["IN", 1442080800000, 0, -142, null, 1442016000000, -142, 12091, 1, 2, 0.047619047619047616, 2, 0.09090909090909091] + - ["IN", 1442016000000, 1, -141, 1442023200000, 1442095200000, -142, 12091, 1, 3, 0.09523809523809523, 3, 0.13636363636363635] + - ["IN", 1442095200000, 4, -137, 1442080800000, 1442019600000, -142, 12091, 1, 4, 0.14285714285714285, 4, 0.18181818181818182] + - ["IN", 1442019600000, 38, -99, 1442016000000, 1442041200000, -142, 12091, 1, 5, 0.19047619047619047, 5, 0.22727272727272727] + - ["IN", 1442041200000, 80, -19, 1442095200000, 1442066400000, -142, 12091, 1, 6, 0.23809523809523808, 6, 0.2727272727272727] + - ["IN", 1442066400000, 116, 97, 1442019600000, 1442088000000, -142, 12091, 1, 7, 0.2857142857142857, 7, 0.3181818181818182] + - ["IN", 1442088000000, 121, 218, 1442041200000, 1442037600000, -142, 12091, 1, 8, 0.3333333333333333, 8, 0.36363636363636365] + - ["IN", 1442037600000, 135, 353, 1442066400000, 1442055600000, -142, 12091, 2, 9, 0.38095238095238093, 9, 0.4090909090909091] + - ["IN", 1442055600000, 166, 519, 1442088000000, 1442084400000, -142, 12091, 2, 10, 0.42857142857142855, 10, 0.45454545454545453] + - ["IN", 1442084400000, 187, 706, 1442037600000, 1442048400000, -142, 12091, 2, 11, 0.47619047619047616, 11, 0.5] + - ["IN", 1442048400000, 262, 968, 1442055600000, 1442052000000, -142, 12091, 2, 12, 0.5238095238095238, 12, 0.5454545454545454] + - ["IN", 1442052000000, 534, 1502, 1442084400000, 1442059200000, -142, 12091, 2, 13, 0.5714285714285714, 13, 0.5909090909090909] + - ["IN", 1442059200000, 708, 2210, 1442048400000, 1442026800000, -142, 12091, 2, 14, 0.6190476190476191, 14, 0.6363636363636364] + - ["IN", 1442026800000, 974, 3184, 1442052000000, 1442073600000, -142, 12091, 2, 15, 0.6666666666666666, 15, 0.6818181818181818] + - ["IN", 1442073600000, 1170, 4354, 1442059200000, 1442034000000, -142, 12091, 3, 16, 0.7142857142857143, 16, 0.7272727272727273] + - ["IN", 1442034000000, 1350, 5704, 1442026800000, 1442030400000, -142, 12091, 3, 17, 0.7619047619047619, 17, 0.7727272727272727] + - ["IN", 1442030400000, 1448, 7152, 1442073600000, 1442062800000, -142, 12091, 3, 18, 0.8095238095238095, 18, 0.8181818181818182] + - ["IN", 1442062800000, 1547, 8699, 1442034000000, 1442044800000, -142, 12091, 3, 19, 0.8571428571428571, 19, 0.8636363636363636] + - ["IN", 1442044800000, 2677, 11376, 1442030400000, 1442077200000, -142, 12091, 3, 20, 0.9047619047619048, 20, 0.9090909090909091] + - ["IN", 1442077200000, 5699, 17075, 1442062800000, 1442070000000, -142, 12091, 3, 21, 0.9523809523809523, 21, 0.9545454545454546] + - ["IN", 1442070000000, 12091, 29166, 1442044800000, null, -142, 12091, 3, 22, 1.0, 22, 1.0] + - ["IQ", 1442095200000, -2, -2, null, 1442041200000, -2, 6, 1, 1, 0.0, 1, 0.25] + - ["IQ", 1442041200000, -1, -3, null, 1442052000000, -2, 6, 1, 2, 0.3333333333333333, 2, 0.5] + - ["IQ", 1442052000000, 0, -3, 1442095200000, 1442044800000, -2, 6, 2, 3, 0.6666666666666666, 3, 0.75] + - ["IQ", 1442044800000, 6, 3, 1442041200000, null, -2, 6, 3, 4, 1.0, 4, 1.0] + - ["IR", 1442073600000, -193, -193, null, 1442055600000, -193, 1455, 1, 1, 0.0, 1, 0.07692307692307693] + - ["IR", 1442055600000, -124, -317, null, 1442041200000, -193, 1455, 1, 2, 0.08333333333333333, 2, 0.15384615384615385] + - ["IR", 1442041200000, -79, -396, 1442073600000, 1442077200000, -193, 1455, 1, 3, 0.16666666666666666, 3, 0.23076923076923078] + - ["IR", 1442077200000, -34, -430, 1442055600000, 1442034000000, -193, 1455, 1, 4, 0.25, 4, 0.3076923076923077] + - ["IR", 1442034000000, -8, -438, 1442041200000, 1442026800000, -193, 1455, 1, 5, 0.3333333333333333, 5, 0.38461538461538464] + - ["IR", 1442026800000, 0, -438, 1442077200000, 1442091600000, -193, 1455, 2, 6, 0.4166666666666667, 6, 0.5384615384615384] + - ["IR", 1442091600000, 0, -438, 1442034000000, 1442080800000, -193, 1455, 2, 6, 0.4166666666666667, 6, 0.5384615384615384] + - ["IR", 1442080800000, 131, -307, 1442026800000, 1442052000000, -193, 1455, 2, 8, 0.5833333333333334, 7, 0.6153846153846154] + - ["IR", 1442052000000, 155, -152, 1442091600000, 1442044800000, -193, 1455, 2, 9, 0.6666666666666666, 8, 0.6923076923076923] + - ["IR", 1442044800000, 306, 154, 1442080800000, 1442030400000, -193, 1455, 3, 10, 0.75, 9, 0.7692307692307693] + - ["IR", 1442030400000, 375, 529, 1442052000000, 1442088000000, -193, 1455, 3, 11, 0.8333333333333334, 10, 0.8461538461538461] + - ["IR", 1442088000000, 714, 1243, 1442044800000, 1442059200000, -193, 1455, 3, 12, 0.9166666666666666, 11, 0.9230769230769231] + - ["IR", 1442059200000, 1455, 2698, 1442030400000, null, -193, 1455, 3, 13, 1.0, 12, 1.0] + - ["IT", 1442030400000, -17, -17, null, 1442037600000, -17, 6240, 1, 1, 0.0, 1, 0.041666666666666664] + - ["IT", 1442037600000, -9, -26, null, 1442016000000, -17, 6240, 1, 2, 0.043478260869565216, 2, 0.08333333333333333] + - ["IT", 1442016000000, 0, -26, 1442030400000, 1442041200000, -17, 6240, 1, 3, 0.08695652173913043, 3, 0.125] + - ["IT", 1442041200000, 20, -6, 1442037600000, 1442070000000, -17, 6240, 1, 4, 0.13043478260869565, 4, 0.16666666666666666] + - ["IT", 1442070000000, 81, 75, 1442016000000, 1442023200000, -17, 6240, 1, 5, 0.17391304347826086, 5, 0.20833333333333334] + - ["IT", 1442023200000, 111, 186, 1442041200000, 1442019600000, -17, 6240, 1, 6, 0.21739130434782608, 6, 0.25] + - ["IT", 1442019600000, 183, 369, 1442070000000, 1442026800000, -17, 6240, 1, 7, 0.2608695652173913, 7, 0.2916666666666667] + - ["IT", 1442026800000, 222, 591, 1442023200000, 1442091600000, -17, 6240, 1, 8, 0.30434782608695654, 8, 0.3333333333333333] + - ["IT", 1442091600000, 351, 942, 1442019600000, 1442059200000, -17, 6240, 2, 9, 0.34782608695652173, 9, 0.375] + - ["IT", 1442059200000, 542, 1484, 1442026800000, 1442098800000, -17, 6240, 2, 10, 0.391304347826087, 10, 0.4166666666666667] + - ["IT", 1442098800000, 565, 2049, 1442091600000, 1442048400000, -17, 6240, 2, 11, 0.43478260869565216, 11, 0.4583333333333333] + - ["IT", 1442048400000, 676, 2725, 1442059200000, 1442034000000, -17, 6240, 2, 12, 0.4782608695652174, 12, 0.5] + - ["IT", 1442034000000, 1006, 3731, 1442098800000, 1442044800000, -17, 6240, 2, 13, 0.5217391304347826, 13, 0.5416666666666666] + - ["IT", 1442044800000, 1483, 5214, 1442048400000, 1442052000000, -17, 6240, 2, 14, 0.5652173913043478, 14, 0.5833333333333334] + - ["IT", 1442052000000, 1880, 7094, 1442034000000, 1442062800000, -17, 6240, 2, 15, 0.6086956521739131, 15, 0.625] + - ["IT", 1442062800000, 1938, 9032, 1442044800000, 1442077200000, -17, 6240, 2, 16, 0.6521739130434783, 16, 0.6666666666666666] + - ["IT", 1442077200000, 2188, 11220, 1442052000000, 1442073600000, -17, 6240, 3, 17, 0.6956521739130435, 17, 0.7083333333333334] + - ["IT", 1442073600000, 2586, 13806, 1442062800000, 1442084400000, -17, 6240, 3, 18, 0.7391304347826086, 18, 0.75] + - ["IT", 1442084400000, 2660, 16466, 1442077200000, 1442095200000, -17, 6240, 3, 19, 0.782608695652174, 19, 0.7916666666666666] + - ["IT", 1442095200000, 2940, 19406, 1442073600000, 1442088000000, -17, 6240, 3, 20, 0.8260869565217391, 20, 0.8333333333333334] + - ["IT", 1442088000000, 3746, 23152, 1442084400000, 1442066400000, -17, 6240, 3, 21, 0.8695652173913043, 21, 0.875] + - ["IT", 1442066400000, 4155, 27307, 1442095200000, 1442080800000, -17, 6240, 3, 22, 0.9130434782608695, 22, 0.9166666666666666] + - ["IT", 1442080800000, 5544, 32851, 1442088000000, 1442055600000, -17, 6240, 3, 23, 0.9565217391304348, 23, 0.9583333333333334] + - ["IT", 1442055600000, 6240, 39091, 1442066400000, null, -17, 6240, 3, 24, 1.0, 24, 1.0] + - ["JM", 1442070000000, 30, 30, null, null, 30, 30, 1, 1, 0.0, 1, 1.0] + - ["JO", 1442055600000, -2, -2, null, 1442059200000, -2, 4, 1, 1, 0.0, 1, 0.3333333333333333] + - ["JO", 1442059200000, 0, -2, null, 1442080800000, -2, 4, 2, 2, 0.5, 2, 0.6666666666666666] + - ["JO", 1442080800000, 4, 2, 1442055600000, null, -2, 4, 3, 3, 1.0, 3, 1.0] + - ["JP", 1442016000000, -113, -113, null, 1442059200000, -113, 2789, 1, 1, 0.0, 1, 0.041666666666666664] + - ["JP", 1442059200000, -85, -198, null, 1442098800000, -113, 2789, 1, 2, 0.043478260869565216, 2, 0.08333333333333333] + - ["JP", 1442098800000, -6, -204, 1442016000000, 1442095200000, -113, 2789, 1, 3, 0.08695652173913043, 3, 0.125] + - ["JP", 1442095200000, 0, -204, 1442059200000, 1442084400000, -113, 2789, 1, 4, 0.13043478260869565, 4, 0.16666666666666666] + - ["JP", 1442084400000, 13, -191, 1442098800000, 1442077200000, -113, 2789, 1, 5, 0.17391304347826086, 5, 0.20833333333333334] + - ["JP", 1442077200000, 51, -140, 1442095200000, 1442088000000, -113, 2789, 1, 6, 0.21739130434782608, 6, 0.25] + - ["JP", 1442088000000, 57, -83, 1442084400000, 1442070000000, -113, 2789, 1, 7, 0.2608695652173913, 7, 0.2916666666666667] + - ["JP", 1442070000000, 79, -4, 1442077200000, 1442066400000, -113, 2789, 1, 8, 0.30434782608695654, 8, 0.3333333333333333] + - ["JP", 1442066400000, 167, 163, 1442088000000, 1442091600000, -113, 2789, 2, 9, 0.34782608695652173, 9, 0.375] + - ["JP", 1442091600000, 228, 391, 1442070000000, 1442080800000, -113, 2789, 2, 10, 0.391304347826087, 10, 0.4166666666666667] + - ["JP", 1442080800000, 420, 811, 1442066400000, 1442062800000, -113, 2789, 2, 11, 0.43478260869565216, 11, 0.4583333333333333] + - ["JP", 1442062800000, 803, 1614, 1442091600000, 1442030400000, -113, 2789, 2, 12, 0.4782608695652174, 12, 0.5] + - ["JP", 1442030400000, 805, 2419, 1442080800000, 1442034000000, -113, 2789, 2, 13, 0.5217391304347826, 13, 0.5416666666666666] + - ["JP", 1442034000000, 910, 3329, 1442062800000, 1442055600000, -113, 2789, 2, 14, 0.5652173913043478, 14, 0.5833333333333334] + - ["JP", 1442055600000, 998, 4327, 1442030400000, 1442026800000, -113, 2789, 2, 15, 0.6086956521739131, 15, 0.625] + - ["JP", 1442026800000, 1035, 5362, 1442034000000, 1442073600000, -113, 2789, 2, 16, 0.6521739130434783, 16, 0.6666666666666666] + - ["JP", 1442073600000, 1162, 6524, 1442055600000, 1442041200000, -113, 2789, 3, 17, 0.6956521739130435, 17, 0.7083333333333334] + - ["JP", 1442041200000, 1373, 7897, 1442026800000, 1442044800000, -113, 2789, 3, 18, 0.7391304347826086, 18, 0.75] + - ["JP", 1442044800000, 1569, 9466, 1442073600000, 1442023200000, -113, 2789, 3, 19, 0.782608695652174, 19, 0.7916666666666666] + - ["JP", 1442023200000, 1959, 11425, 1442041200000, 1442048400000, -113, 2789, 3, 20, 0.8260869565217391, 20, 0.8333333333333334] + - ["JP", 1442048400000, 1981, 13406, 1442044800000, 1442019600000, -113, 2789, 3, 21, 0.8695652173913043, 21, 0.875] + - ["JP", 1442019600000, 2002, 15408, 1442023200000, 1442037600000, -113, 2789, 3, 22, 0.9130434782608695, 22, 0.9166666666666666] + - ["JP", 1442037600000, 2181, 17589, 1442048400000, 1442052000000, -113, 2789, 3, 23, 0.9565217391304348, 23, 0.9583333333333334] + - ["JP", 1442052000000, 2789, 20378, 1442019600000, null, -113, 2789, 3, 24, 1.0, 24, 1.0] + - ["KE", 1442044800000, -1, -1, null, null, -1, -1, 1, 1, 0.0, 1, 1.0] + - ["KG", 1442073600000, 6, 6, null, null, 6, 6, 1, 1, 0.0, 1, 1.0] + - ["KR", 1442048400000, -374, -374, null, 1442026800000, -374, 3640, 1, 1, 0.0, 1, 0.045454545454545456] + - ["KR", 1442026800000, -179, -553, null, 1442077200000, -374, 3640, 1, 2, 0.047619047619047616, 2, 0.09090909090909091] + - ["KR", 1442077200000, -40, -593, 1442048400000, 1442098800000, -374, 3640, 1, 3, 0.09523809523809523, 3, 0.13636363636363635] + - ["KR", 1442098800000, -36, -629, 1442026800000, 1442080800000, -374, 3640, 1, 4, 0.14285714285714285, 4, 0.18181818181818182] + - ["KR", 1442080800000, -33, -662, 1442077200000, 1442052000000, -374, 3640, 1, 5, 0.19047619047619047, 5, 0.22727272727272727] + - ["KR", 1442052000000, -3, -665, 1442098800000, 1442041200000, -374, 3640, 1, 6, 0.23809523809523808, 6, 0.2727272727272727] + - ["KR", 1442041200000, 20, -645, 1442080800000, 1442037600000, -374, 3640, 1, 7, 0.2857142857142857, 7, 0.3181818181818182] + - ["KR", 1442037600000, 26, -619, 1442052000000, 1442059200000, -374, 3640, 1, 8, 0.3333333333333333, 8, 0.36363636363636365] + - ["KR", 1442059200000, 208, -411, 1442041200000, 1442070000000, -374, 3640, 2, 9, 0.38095238095238093, 9, 0.4090909090909091] + - ["KR", 1442070000000, 222, -189, 1442037600000, 1442084400000, -374, 3640, 2, 10, 0.42857142857142855, 10, 0.45454545454545453] + - ["KR", 1442084400000, 314, 125, 1442059200000, 1442023200000, -374, 3640, 2, 11, 0.47619047619047616, 11, 0.5] + - ["KR", 1442023200000, 319, 444, 1442070000000, 1442034000000, -374, 3640, 2, 12, 0.5238095238095238, 12, 0.5454545454545454] + - ["KR", 1442034000000, 434, 878, 1442084400000, 1442019600000, -374, 3640, 2, 13, 0.5714285714285714, 13, 0.5909090909090909] + - ["KR", 1442019600000, 445, 1323, 1442023200000, 1442088000000, -374, 3640, 2, 14, 0.6190476190476191, 14, 0.6363636363636364] + - ["KR", 1442088000000, 524, 1847, 1442034000000, 1442095200000, -374, 3640, 2, 15, 0.6666666666666666, 15, 0.6818181818181818] + - ["KR", 1442095200000, 827, 2674, 1442019600000, 1442044800000, -374, 3640, 3, 16, 0.7142857142857143, 16, 0.7272727272727273] + - ["KR", 1442044800000, 829, 3503, 1442088000000, 1442016000000, -374, 3640, 3, 17, 0.7619047619047619, 17, 0.7727272727272727] + - ["KR", 1442016000000, 1024, 4527, 1442095200000, 1442030400000, -374, 3640, 3, 18, 0.8095238095238095, 18, 0.8181818181818182] + - ["KR", 1442030400000, 1035, 5562, 1442044800000, 1442062800000, -374, 3640, 3, 19, 0.8571428571428571, 19, 0.8636363636363636] + - ["KR", 1442062800000, 1096, 6658, 1442016000000, 1442066400000, -374, 3640, 3, 20, 0.9047619047619048, 20, 0.9090909090909091] + - ["KR", 1442066400000, 3299, 9957, 1442030400000, 1442055600000, -374, 3640, 3, 21, 0.9523809523809523, 21, 0.9545454545454546] + - ["KR", 1442055600000, 3640, 13597, 1442062800000, null, -374, 3640, 3, 22, 1.0, 22, 1.0] + - ["KW", 1442080800000, -33, -33, null, 1442055600000, -33, 1815, 1, 1, 0.0, 1, 0.25] + - ["KW", 1442055600000, -2, -35, null, 1442077200000, -33, 1815, 1, 2, 0.3333333333333333, 2, 0.75] + - ["KW", 1442077200000, -2, -37, 1442080800000, 1442070000000, -33, 1815, 2, 2, 0.3333333333333333, 2, 0.75] + - ["KW", 1442070000000, 1815, 1778, 1442055600000, null, -33, 1815, 3, 4, 1.0, 3, 1.0] + - ["KZ", 1442077200000, -317, -317, null, 1442084400000, -317, 439, 1, 1, 0.0, 1, 0.09090909090909091] + - ["KZ", 1442084400000, -22, -339, null, 1442062800000, -317, 439, 1, 2, 0.1, 2, 0.18181818181818182] + - ["KZ", 1442062800000, 0, -339, 1442077200000, 1442066400000, -317, 439, 1, 3, 0.2, 3, 0.36363636363636365] + - ["KZ", 1442066400000, 0, -339, 1442084400000, 1442059200000, -317, 439, 1, 3, 0.2, 3, 0.36363636363636365] + - ["KZ", 1442059200000, 33, -306, 1442062800000, 1442055600000, -317, 439, 2, 5, 0.4, 4, 0.45454545454545453] + - ["KZ", 1442055600000, 63, -243, 1442066400000, 1442095200000, -317, 439, 2, 6, 0.5, 5, 0.5454545454545454] + - ["KZ", 1442095200000, 91, -152, 1442059200000, 1442034000000, -317, 439, 2, 7, 0.6, 6, 0.6363636363636364] + - ["KZ", 1442034000000, 161, 9, 1442055600000, 1442044800000, -317, 439, 2, 8, 0.7, 7, 0.7272727272727273] + - ["KZ", 1442044800000, 401, 410, 1442095200000, 1442052000000, -317, 439, 3, 9, 0.8, 8, 0.8181818181818182] + - ["KZ", 1442052000000, 412, 822, 1442034000000, 1442048400000, -317, 439, 3, 10, 0.9, 9, 0.9090909090909091] + - ["KZ", 1442048400000, 439, 1261, 1442044800000, null, -317, 439, 3, 11, 1.0, 10, 1.0] + - ["LB", 1442055600000, -67, -67, null, null, -67, -67, 1, 1, 0.0, 1, 1.0] + - ["LK", 1442084400000, -3, -3, null, 1442048400000, -3, 79, 1, 1, 0.0, 1, 0.25] + - ["LK", 1442048400000, 8, 5, null, 1442052000000, -3, 79, 1, 2, 0.3333333333333333, 2, 0.5] + - ["LK", 1442052000000, 47, 52, 1442084400000, 1442026800000, -3, 79, 2, 3, 0.6666666666666666, 3, 0.75] + - ["LK", 1442026800000, 79, 131, 1442048400000, null, -3, 79, 3, 4, 1.0, 4, 1.0] + - ["LT", 1442098800000, -24, -24, null, 1442080800000, -24, 12, 1, 1, 0.0, 1, 0.5] + - ["LT", 1442080800000, 12, -12, null, null, -24, 12, 2, 2, 1.0, 2, 1.0] + - ["LU", 1442066400000, 0, 0, null, 1442095200000, 0, 525, 1, 1, 0.0, 1, 0.25] + - ["LU", 1442095200000, 2, 2, null, 1442059200000, 0, 525, 1, 2, 0.3333333333333333, 2, 0.5] + - ["LU", 1442059200000, 79, 81, 1442066400000, 1442077200000, 0, 525, 2, 3, 0.6666666666666666, 3, 0.75] + - ["LU", 1442077200000, 525, 606, 1442095200000, null, 0, 525, 3, 4, 1.0, 4, 1.0] + - ["LV", 1442095200000, 0, 0, null, null, 0, 0, 1, 1, 0.0, 1, 1.0] + - ["MA", 1442059200000, -56, -56, null, 1442019600000, -56, 250, 1, 1, 0.0, 1, 0.14285714285714285] + - ["MA", 1442019600000, -1, -57, null, 1442062800000, -56, 250, 1, 2, 0.16666666666666666, 2, 0.2857142857142857] + - ["MA", 1442062800000, 0, -57, 1442059200000, 1442080800000, -56, 250, 1, 3, 0.3333333333333333, 3, 0.42857142857142855] + - ["MA", 1442080800000, 5, -52, 1442019600000, 1442098800000, -56, 250, 2, 4, 0.5, 4, 0.5714285714285714] + - ["MA", 1442098800000, 8, -44, 1442062800000, 1442055600000, -56, 250, 2, 5, 0.6666666666666666, 5, 0.7142857142857143] + - ["MA", 1442055600000, 23, -21, 1442080800000, 1442077200000, -56, 250, 3, 6, 0.8333333333333334, 6, 0.8571428571428571] + - ["MA", 1442077200000, 250, 229, 1442098800000, null, -56, 250, 3, 7, 1.0, 7, 1.0] + - ["MD", 1442077200000, 6916, 6916, null, null, 6916, 6916, 1, 1, 0.0, 1, 1.0] + - ["ME", 1442073600000, 0, 0, null, null, 0, 0, 1, 1, 0.0, 1, 1.0] + - ["MH", 1442052000000, 40, 40, null, null, 40, 40, 1, 1, 0.0, 1, 1.0] + - ["MK", 1442077200000, -72, -72, null, null, -72, -72, 1, 1, 0.0, 1, 1.0] + - ["MM", 1442070000000, 3, 3, null, 1442073600000, 3, 25, 1, 1, 0.0, 1, 0.5] + - ["MM", 1442073600000, 25, 28, null, null, 3, 25, 2, 2, 1.0, 2, 1.0] + - ["MO", 1442070000000, 18, 18, null, 1442034000000, 18, 30, 1, 1, 0.0, 1, 0.5] + - ["MO", 1442034000000, 30, 48, null, null, 18, 30, 2, 2, 1.0, 2, 1.0] + - ["MR", 1442080800000, 10, 10, null, null, 10, 10, 1, 1, 0.0, 1, 1.0] + - ["MT", 1442048400000, -1, -1, null, null, -1, -1, 1, 1, 0.0, 1, 1.0] + - ["MV", 1442073600000, -3, -3, null, null, -3, -3, 1, 1, 0.0, 1, 1.0] + - ["MX", 1442095200000, -456, -456, null, 1442080800000, -456, 3874, 1, 1, 0.0, 1, 0.058823529411764705] + - ["MX", 1442080800000, -376, -832, null, 1442041200000, -456, 3874, 1, 2, 0.0625, 2, 0.11764705882352941] + - ["MX", 1442041200000, -294, -1126, 1442095200000, 1442016000000, -456, 3874, 1, 3, 0.125, 3, 0.17647058823529413] + - ["MX", 1442016000000, -67, -1193, 1442080800000, 1442073600000, -456, 3874, 1, 4, 0.1875, 4, 0.23529411764705882] + - ["MX", 1442073600000, -21, -1214, 1442041200000, 1442066400000, -456, 3874, 1, 5, 0.25, 5, 0.29411764705882354] + - ["MX", 1442066400000, -1, -1215, 1442016000000, 1442070000000, -456, 3874, 1, 6, 0.3125, 6, 0.4117647058823529] + - ["MX", 1442070000000, -1, -1216, 1442073600000, 1442037600000, -456, 3874, 2, 6, 0.3125, 6, 0.4117647058823529] + - ["MX", 1442037600000, 4, -1212, 1442066400000, 1442098800000, -456, 3874, 2, 8, 0.4375, 7, 0.47058823529411764] + - ["MX", 1442098800000, 28, -1184, 1442070000000, 1442030400000, -456, 3874, 2, 9, 0.5, 8, 0.5294117647058824] + - ["MX", 1442030400000, 373, -811, 1442037600000, 1442088000000, -456, 3874, 2, 10, 0.5625, 9, 0.5882352941176471] + - ["MX", 1442088000000, 494, -317, 1442098800000, 1442023200000, -456, 3874, 2, 11, 0.625, 10, 0.6470588235294118] + - ["MX", 1442023200000, 549, 232, 1442030400000, 1442091600000, -456, 3874, 2, 12, 0.6875, 11, 0.7058823529411765] + - ["MX", 1442091600000, 799, 1031, 1442088000000, 1442034000000, -456, 3874, 3, 13, 0.75, 12, 0.7647058823529411] + - ["MX", 1442034000000, 944, 1975, 1442023200000, 1442084400000, -456, 3874, 3, 14, 0.8125, 13, 0.8235294117647058] + - ["MX", 1442084400000, 981, 2956, 1442091600000, 1442026800000, -456, 3874, 3, 15, 0.875, 14, 0.8823529411764706] + - ["MX", 1442026800000, 3642, 6598, 1442034000000, 1442077200000, -456, 3874, 3, 16, 0.9375, 15, 0.9411764705882353] + - ["MX", 1442077200000, 3874, 10472, 1442084400000, null, -456, 3874, 3, 17, 1.0, 16, 1.0] + - ["MY", 1442044800000, -127, -127, null, 1442077200000, -127, 1028, 1, 1, 0.0, 1, 0.08333333333333333] + - ["MY", 1442077200000, -10, -137, null, 1442019600000, -127, 1028, 1, 2, 0.09090909090909091, 2, 0.16666666666666666] + - ["MY", 1442019600000, -7, -144, 1442044800000, 1442030400000, -127, 1028, 1, 3, 0.18181818181818182, 3, 0.25] + - ["MY", 1442030400000, -3, -147, 1442077200000, 1442059200000, -127, 1028, 1, 4, 0.2727272727272727, 4, 0.3333333333333333] + - ["MY", 1442059200000, 0, -147, 1442019600000, 1442055600000, -127, 1028, 2, 5, 0.36363636363636365, 5, 0.4166666666666667] + - ["MY", 1442055600000, 1, -146, 1442030400000, 1442066400000, -127, 1028, 2, 6, 0.45454545454545453, 6, 0.6666666666666666] + - ["MY", 1442066400000, 1, -145, 1442059200000, 1442073600000, -127, 1028, 2, 6, 0.45454545454545453, 6, 0.6666666666666666] + - ["MY", 1442073600000, 1, -144, 1442055600000, 1442048400000, -127, 1028, 2, 6, 0.45454545454545453, 6, 0.6666666666666666] + - ["MY", 1442048400000, 649, 505, 1442066400000, 1442098800000, -127, 1028, 3, 9, 0.7272727272727273, 7, 0.75] + - ["MY", 1442098800000, 739, 1244, 1442073600000, 1442041200000, -127, 1028, 3, 10, 0.8181818181818182, 8, 0.8333333333333334] + - ["MY", 1442041200000, 935, 2179, 1442048400000, 1442034000000, -127, 1028, 3, 11, 0.9090909090909091, 9, 0.9166666666666666] + - ["MY", 1442034000000, 1028, 3207, 1442098800000, null, -127, 1028, 3, 12, 1.0, 10, 1.0] + - ["NG", 1442070000000, 6, 6, null, 1442052000000, 6, 208, 1, 1, 0.0, 1, 0.5] + - ["NG", 1442052000000, 208, 214, null, null, 6, 208, 2, 2, 1.0, 2, 1.0] + - ["NL", 1442070000000, -84, -84, null, 1442062800000, -84, 8947, 1, 1, 0.0, 1, 0.058823529411764705] + - ["NL", 1442062800000, -30, -114, null, 1442034000000, -84, 8947, 1, 2, 0.0625, 2, 0.11764705882352941] + - ["NL", 1442034000000, 0, -114, 1442070000000, 1442098800000, -84, 8947, 1, 3, 0.125, 3, 0.17647058823529413] + - ["NL", 1442098800000, 4, -110, 1442062800000, 1442088000000, -84, 8947, 1, 4, 0.1875, 4, 0.23529411764705882] + - ["NL", 1442088000000, 12, -98, 1442034000000, 1442044800000, -84, 8947, 1, 5, 0.25, 5, 0.29411764705882354] + - ["NL", 1442044800000, 16, -82, 1442098800000, 1442091600000, -84, 8947, 1, 6, 0.3125, 6, 0.35294117647058826] + - ["NL", 1442091600000, 19, -63, 1442088000000, 1442052000000, -84, 8947, 2, 7, 0.375, 7, 0.4117647058823529] + - ["NL", 1442052000000, 53, -10, 1442044800000, 1442066400000, -84, 8947, 2, 8, 0.4375, 8, 0.47058823529411764] + - ["NL", 1442066400000, 61, 51, 1442091600000, 1442095200000, -84, 8947, 2, 9, 0.5, 9, 0.5294117647058824] + - ["NL", 1442095200000, 70, 121, 1442052000000, 1442055600000, -84, 8947, 2, 10, 0.5625, 10, 0.5882352941176471] + - ["NL", 1442055600000, 105, 226, 1442066400000, 1442073600000, -84, 8947, 2, 11, 0.625, 11, 0.6470588235294118] + - ["NL", 1442073600000, 166, 392, 1442095200000, 1442059200000, -84, 8947, 2, 12, 0.6875, 12, 0.7058823529411765] + - ["NL", 1442059200000, 206, 598, 1442055600000, 1442084400000, -84, 8947, 3, 13, 0.75, 13, 0.7647058823529411] + - ["NL", 1442084400000, 436, 1034, 1442073600000, 1442077200000, -84, 8947, 3, 14, 0.8125, 14, 0.8235294117647058] + - ["NL", 1442077200000, 878, 1912, 1442059200000, 1442048400000, -84, 8947, 3, 15, 0.875, 15, 0.8823529411764706] + - ["NL", 1442048400000, 1303, 3215, 1442084400000, 1442080800000, -84, 8947, 3, 16, 0.9375, 16, 0.9411764705882353] + - ["NL", 1442080800000, 8947, 12162, 1442077200000, null, -84, 8947, 3, 17, 1.0, 17, 1.0] + - ["NO", 1442048400000, -447, -447, null, 1442095200000, -447, 447, 1, 1, 0.0, 1, 0.09090909090909091] + - ["NO", 1442095200000, -1, -448, null, 1442098800000, -447, 447, 1, 2, 0.1, 2, 0.18181818181818182] + - ["NO", 1442098800000, 2, -446, 1442048400000, 1442088000000, -447, 447, 1, 3, 0.2, 3, 0.2727272727272727] + - ["NO", 1442088000000, 15, -431, 1442095200000, 1442091600000, -447, 447, 1, 4, 0.3, 4, 0.45454545454545453] + - ["NO", 1442091600000, 15, -416, 1442098800000, 1442055600000, -447, 447, 2, 4, 0.3, 4, 0.45454545454545453] + - ["NO", 1442055600000, 29, -387, 1442088000000, 1442080800000, -447, 447, 2, 6, 0.5, 5, 0.5454545454545454] + - ["NO", 1442080800000, 31, -356, 1442091600000, 1442019600000, -447, 447, 2, 7, 0.6, 6, 0.6363636363636364] + - ["NO", 1442019600000, 48, -308, 1442055600000, 1442066400000, -447, 447, 2, 8, 0.7, 7, 0.7272727272727273] + - ["NO", 1442066400000, 71, -237, 1442080800000, 1442073600000, -447, 447, 3, 9, 0.8, 8, 0.8181818181818182] + - ["NO", 1442073600000, 222, -15, 1442019600000, 1442052000000, -447, 447, 3, 10, 0.9, 9, 0.9090909090909091] + - ["NO", 1442052000000, 447, 432, 1442066400000, null, -447, 447, 3, 11, 1.0, 10, 1.0] + - ["NP", 1442048400000, 61, 61, null, null, 61, 61, 1, 1, 0.0, 1, 1.0] + - ["NZ", 1442084400000, -52, -52, null, 1442095200000, -52, 635, 1, 1, 0.0, 1, 0.1111111111111111] + - ["NZ", 1442095200000, -4, -56, null, 1442098800000, -52, 635, 1, 2, 0.125, 2, 0.2222222222222222] + - ["NZ", 1442098800000, -2, -58, 1442084400000, 1442019600000, -52, 635, 1, 3, 0.25, 3, 0.3333333333333333] + - ["NZ", 1442019600000, 28, -30, 1442095200000, 1442037600000, -52, 635, 2, 4, 0.375, 4, 0.4444444444444444] + - ["NZ", 1442037600000, 66, 36, 1442098800000, 1442048400000, -52, 635, 2, 5, 0.5, 5, 0.5555555555555556] + - ["NZ", 1442048400000, 189, 225, 1442019600000, 1442088000000, -52, 635, 2, 6, 0.625, 6, 0.6666666666666666] + - ["NZ", 1442088000000, 405, 630, 1442037600000, 1442059200000, -52, 635, 3, 7, 0.75, 7, 0.7777777777777778] + - ["NZ", 1442059200000, 428, 1058, 1442048400000, 1442026800000, -52, 635, 3, 8, 0.875, 8, 0.8888888888888888] + - ["NZ", 1442026800000, 635, 1693, 1442088000000, null, -52, 635, 3, 9, 1.0, 9, 1.0] + - ["OM", 1442052000000, 0, 0, null, null, 0, 0, 1, 1, 0.0, 1, 1.0] + - ["PA", 1442026800000, 0, 0, null, null, 0, 0, 1, 1, 0.0, 1, 1.0] + - ["PE", 1442077200000, -163, -163, null, 1442084400000, -163, 1861, 1, 1, 0.0, 1, 0.1111111111111111] + - ["PE", 1442084400000, -68, -231, null, 1442095200000, -163, 1861, 1, 2, 0.125, 2, 0.2222222222222222] + - ["PE", 1442095200000, -19, -250, 1442077200000, 1442026800000, -163, 1861, 1, 3, 0.25, 3, 0.3333333333333333] + - ["PE", 1442026800000, -12, -262, 1442084400000, 1442062800000, -163, 1861, 2, 4, 0.375, 4, 0.5555555555555556] + - ["PE", 1442062800000, -12, -274, 1442095200000, 1442080800000, -163, 1861, 2, 4, 0.375, 4, 0.5555555555555556] + - ["PE", 1442080800000, -2, -276, 1442026800000, 1442023200000, -163, 1861, 2, 6, 0.625, 5, 0.6666666666666666] + - ["PE", 1442023200000, 26, -250, 1442062800000, 1442019600000, -163, 1861, 3, 7, 0.75, 6, 0.7777777777777778] + - ["PE", 1442019600000, 523, 273, 1442080800000, 1442098800000, -163, 1861, 3, 8, 0.875, 7, 0.8888888888888888] + - ["PE", 1442098800000, 1861, 2134, 1442023200000, null, -163, 1861, 3, 9, 1.0, 8, 1.0] + - ["PH", 1442073600000, -227, -227, null, 1442041200000, -227, 1969, 1, 1, 0.0, 1, 0.047619047619047616] + - ["PH", 1442041200000, 0, -227, null, 1442077200000, -227, 1969, 1, 2, 0.05, 2, 0.09523809523809523] + - ["PH", 1442077200000, 2, -225, 1442073600000, 1442019600000, -227, 1969, 1, 3, 0.1, 3, 0.14285714285714285] + - ["PH", 1442019600000, 6, -219, 1442041200000, 1442098800000, -227, 1969, 1, 4, 0.15, 4, 0.19047619047619047] + - ["PH", 1442098800000, 8, -211, 1442077200000, 1442037600000, -227, 1969, 1, 5, 0.2, 5, 0.23809523809523808] + - ["PH", 1442037600000, 17, -194, 1442019600000, 1442052000000, -227, 1969, 1, 6, 0.25, 6, 0.2857142857142857] + - ["PH", 1442052000000, 22, -172, 1442098800000, 1442030400000, -227, 1969, 1, 7, 0.3, 7, 0.3333333333333333] + - ["PH", 1442030400000, 26, -146, 1442037600000, 1442080800000, -227, 1969, 2, 8, 0.35, 8, 0.38095238095238093] + - ["PH", 1442080800000, 32, -114, 1442052000000, 1442070000000, -227, 1969, 2, 9, 0.4, 9, 0.42857142857142855] + - ["PH", 1442070000000, 34, -80, 1442030400000, 1442084400000, -227, 1969, 2, 10, 0.45, 10, 0.47619047619047616] + - ["PH", 1442084400000, 39, -41, 1442080800000, 1442044800000, -227, 1969, 2, 11, 0.5, 11, 0.5238095238095238] + - ["PH", 1442044800000, 55, 14, 1442070000000, 1442034000000, -227, 1969, 2, 12, 0.55, 12, 0.5714285714285714] + - ["PH", 1442034000000, 59, 73, 1442084400000, 1442048400000, -227, 1969, 2, 13, 0.6, 13, 0.6190476190476191] + - ["PH", 1442048400000, 62, 135, 1442044800000, 1442062800000, -227, 1969, 2, 14, 0.65, 14, 0.6666666666666666] + - ["PH", 1442062800000, 171, 306, 1442034000000, 1442059200000, -227, 1969, 3, 15, 0.7, 15, 0.7142857142857143] + - ["PH", 1442059200000, 273, 579, 1442048400000, 1442023200000, -227, 1969, 3, 16, 0.75, 16, 0.7619047619047619] + - ["PH", 1442023200000, 459, 1038, 1442062800000, 1442091600000, -227, 1969, 3, 17, 0.8, 17, 0.8095238095238095] + - ["PH", 1442091600000, 816, 1854, 1442059200000, 1442026800000, -227, 1969, 3, 18, 0.85, 18, 0.8571428571428571] + - ["PH", 1442026800000, 910, 2764, 1442023200000, 1442066400000, -227, 1969, 3, 19, 0.9, 19, 0.9047619047619048] + - ["PH", 1442066400000, 1880, 4644, 1442091600000, 1442055600000, -227, 1969, 3, 20, 0.95, 20, 0.9523809523809523] + - ["PH", 1442055600000, 1969, 6613, 1442026800000, null, -227, 1969, 3, 21, 1.0, 21, 1.0] + - ["PK", 1442048400000, 15, 15, null, 1442062800000, 15, 335, 1, 1, 0.0, 1, 0.14285714285714285] + - ["PK", 1442062800000, 23, 38, null, 1442041200000, 15, 335, 1, 2, 0.16666666666666666, 2, 0.2857142857142857] + - ["PK", 1442041200000, 24, 62, 1442048400000, 1442070000000, 15, 335, 1, 3, 0.3333333333333333, 3, 0.42857142857142855] + - ["PK", 1442070000000, 43, 105, 1442062800000, 1442037600000, 15, 335, 2, 4, 0.5, 4, 0.5714285714285714] + - ["PK", 1442037600000, 100, 205, 1442041200000, 1442026800000, 15, 335, 2, 5, 0.6666666666666666, 5, 0.7142857142857143] + - ["PK", 1442026800000, 101, 306, 1442070000000, 1442019600000, 15, 335, 3, 6, 0.8333333333333334, 6, 0.8571428571428571] + - ["PK", 1442019600000, 335, 641, 1442037600000, null, 15, 335, 3, 7, 1.0, 7, 1.0] + - ["PL", 1442098800000, -9, -9, null, 1442080800000, -9, 4171, 1, 1, 0.0, 1, 0.05555555555555555] + - ["PL", 1442080800000, 7, -2, null, 1442084400000, -9, 4171, 1, 2, 0.058823529411764705, 2, 0.1111111111111111] + - ["PL", 1442084400000, 13, 11, 1442098800000, 1442073600000, -9, 4171, 1, 3, 0.11764705882352941, 3, 0.16666666666666666] + - ["PL", 1442073600000, 30, 41, 1442080800000, 1442066400000, -9, 4171, 1, 4, 0.17647058823529413, 4, 0.2222222222222222] + - ["PL", 1442066400000, 34, 75, 1442084400000, 1442037600000, -9, 4171, 1, 5, 0.23529411764705882, 5, 0.2777777777777778] + - ["PL", 1442037600000, 95, 170, 1442073600000, 1442070000000, -9, 4171, 1, 6, 0.29411764705882354, 6, 0.3333333333333333] + - ["PL", 1442070000000, 146, 316, 1442066400000, 1442059200000, -9, 4171, 2, 7, 0.35294117647058826, 7, 0.3888888888888889] + - ["PL", 1442059200000, 199, 515, 1442037600000, 1442041200000, -9, 4171, 2, 8, 0.4117647058823529, 8, 0.4444444444444444] + - ["PL", 1442041200000, 281, 796, 1442070000000, 1442044800000, -9, 4171, 2, 9, 0.47058823529411764, 9, 0.5] + - ["PL", 1442044800000, 319, 1115, 1442059200000, 1442077200000, -9, 4171, 2, 10, 0.5294117647058824, 10, 0.5555555555555556] + - ["PL", 1442077200000, 324, 1439, 1442041200000, 1442052000000, -9, 4171, 2, 11, 0.5882352941176471, 11, 0.6111111111111112] + - ["PL", 1442052000000, 330, 1769, 1442044800000, 1442088000000, -9, 4171, 2, 12, 0.6470588235294118, 12, 0.6666666666666666] + - ["PL", 1442088000000, 346, 2115, 1442077200000, 1442048400000, -9, 4171, 3, 13, 0.7058823529411765, 13, 0.7222222222222222] + - ["PL", 1442048400000, 366, 2481, 1442052000000, 1442055600000, -9, 4171, 3, 14, 0.7647058823529411, 14, 0.7777777777777778] + - ["PL", 1442055600000, 410, 2891, 1442088000000, 1442091600000, -9, 4171, 3, 15, 0.8235294117647058, 15, 0.8333333333333334] + - ["PL", 1442091600000, 902, 3793, 1442048400000, 1442095200000, -9, 4171, 3, 16, 0.8823529411764706, 16, 0.8888888888888888] + - ["PL", 1442095200000, 1851, 5644, 1442055600000, 1442062800000, -9, 4171, 3, 17, 0.9411764705882353, 17, 0.9444444444444444] + - ["PL", 1442062800000, 4171, 9815, 1442091600000, null, -9, 4171, 3, 18, 1.0, 18, 1.0] + - ["PR", 1442059200000, -35, -35, null, 1442030400000, -35, 29, 1, 1, 0.0, 1, 0.2] + - ["PR", 1442030400000, 2, -33, null, 1442077200000, -35, 29, 1, 2, 0.25, 2, 0.4] + - ["PR", 1442077200000, 5, -28, 1442059200000, 1442026800000, -35, 29, 2, 3, 0.5, 3, 0.6] + - ["PR", 1442026800000, 22, -6, 1442030400000, 1442095200000, -35, 29, 2, 4, 0.75, 4, 0.8] + - ["PR", 1442095200000, 29, 23, 1442077200000, null, -35, 29, 3, 5, 1.0, 5, 1.0] + - ["PT", 1442080800000, -79, -79, null, 1442077200000, -79, 3470, 1, 1, 0.0, 1, 0.1] + - ["PT", 1442077200000, -75, -154, null, 1442098800000, -79, 3470, 1, 2, 0.1111111111111111, 2, 0.2] + - ["PT", 1442098800000, 2, -152, 1442080800000, 1442044800000, -79, 3470, 1, 3, 0.2222222222222222, 3, 0.3] + - ["PT", 1442044800000, 11, -141, 1442077200000, 1442066400000, -79, 3470, 1, 4, 0.3333333333333333, 4, 0.4] + - ["PT", 1442066400000, 12, -129, 1442098800000, 1442095200000, -79, 3470, 2, 5, 0.4444444444444444, 5, 0.5] + - ["PT", 1442095200000, 19, -110, 1442044800000, 1442052000000, -79, 3470, 2, 6, 0.5555555555555556, 6, 0.6] + - ["PT", 1442052000000, 102, -8, 1442066400000, 1442019600000, -79, 3470, 2, 7, 0.6666666666666666, 7, 0.7] + - ["PT", 1442019600000, 172, 164, 1442095200000, 1442088000000, -79, 3470, 3, 8, 0.7777777777777778, 8, 0.8] + - ["PT", 1442088000000, 403, 567, 1442052000000, 1442070000000, -79, 3470, 3, 9, 0.8888888888888888, 9, 0.9] + - ["PT", 1442070000000, 3470, 4037, 1442019600000, null, -79, 3470, 3, 10, 1.0, 10, 1.0] + - ["PY", 1442019600000, 1, 1, null, 1442080800000, 1, 628, 1, 1, 0.0, 1, 0.3333333333333333] + - ["PY", 1442080800000, 5, 6, null, 1442084400000, 1, 628, 2, 2, 0.5, 2, 0.6666666666666666] + - ["PY", 1442084400000, 628, 634, 1442019600000, null, 1, 628, 3, 3, 1.0, 3, 1.0] + - ["QA", 1442041200000, 13, 13, null, null, 13, 13, 1, 1, 0.0, 1, 1.0] + - ["RO", 1442070000000, -29, -29, null, 1442091600000, -29, 845, 1, 1, 0.0, 1, 0.1] + - ["RO", 1442091600000, 0, -29, null, 1442073600000, -29, 845, 1, 2, 0.1111111111111111, 2, 0.2] + - ["RO", 1442073600000, 15, -14, 1442070000000, 1442055600000, -29, 845, 1, 3, 0.2222222222222222, 3, 0.3] + - ["RO", 1442055600000, 26, 12, 1442091600000, 1442034000000, -29, 845, 1, 4, 0.3333333333333333, 4, 0.4] + - ["RO", 1442034000000, 68, 80, 1442073600000, 1442044800000, -29, 845, 2, 5, 0.4444444444444444, 5, 0.5] + - ["RO", 1442044800000, 284, 364, 1442055600000, 1442052000000, -29, 845, 2, 6, 0.5555555555555556, 6, 0.6] + - ["RO", 1442052000000, 319, 683, 1442034000000, 1442062800000, -29, 845, 2, 7, 0.6666666666666666, 7, 0.7] + - ["RO", 1442062800000, 541, 1224, 1442044800000, 1442095200000, -29, 845, 3, 8, 0.7777777777777778, 8, 0.8] + - ["RO", 1442095200000, 824, 2048, 1442052000000, 1442041200000, -29, 845, 3, 9, 0.8888888888888888, 9, 0.9] + - ["RO", 1442041200000, 845, 2893, 1442062800000, null, -29, 845, 3, 10, 1.0, 10, 1.0] + - ["RS", 1442091600000, -15, -15, null, 1442066400000, -15, 813, 1, 1, 0.0, 1, 0.14285714285714285] + - ["RS", 1442066400000, 0, -15, null, 1442080800000, -15, 813, 1, 2, 0.16666666666666666, 2, 0.42857142857142855] + - ["RS", 1442080800000, 0, -15, 1442091600000, 1442019600000, -15, 813, 1, 2, 0.16666666666666666, 2, 0.42857142857142855] + - ["RS", 1442019600000, 6, -9, 1442066400000, 1442062800000, -15, 813, 2, 4, 0.5, 3, 0.5714285714285714] + - ["RS", 1442062800000, 13, 4, 1442080800000, 1442084400000, -15, 813, 2, 5, 0.6666666666666666, 4, 0.7142857142857143] + - ["RS", 1442084400000, 89, 93, 1442019600000, 1442073600000, -15, 813, 3, 6, 0.8333333333333334, 5, 0.8571428571428571] + - ["RS", 1442073600000, 813, 906, 1442062800000, null, -15, 813, 3, 7, 1.0, 6, 1.0] + - ["RU", 1442037600000, -324, -324, null, 1442026800000, -324, 12098, 1, 1, 0.0, 1, 0.043478260869565216] + - ["RU", 1442026800000, 0, -324, null, 1442030400000, -324, 12098, 1, 2, 0.045454545454545456, 2, 0.08695652173913043] + - ["RU", 1442030400000, 76, -248, 1442037600000, 1442062800000, -324, 12098, 1, 3, 0.09090909090909091, 3, 0.13043478260869565] + - ["RU", 1442062800000, 168, -80, 1442026800000, 1442023200000, -324, 12098, 1, 4, 0.13636363636363635, 4, 0.17391304347826086] + - ["RU", 1442023200000, 299, 219, 1442030400000, 1442095200000, -324, 12098, 1, 5, 0.18181818181818182, 5, 0.21739130434782608] + - ["RU", 1442095200000, 435, 654, 1442062800000, 1442055600000, -324, 12098, 1, 6, 0.22727272727272727, 6, 0.2608695652173913] + - ["RU", 1442055600000, 499, 1153, 1442023200000, 1442041200000, -324, 12098, 1, 7, 0.2727272727272727, 7, 0.30434782608695654] + - ["RU", 1442041200000, 580, 1733, 1442095200000, 1442080800000, -324, 12098, 1, 8, 0.3181818181818182, 8, 0.34782608695652173] + - ["RU", 1442080800000, 655, 2388, 1442055600000, 1442034000000, -324, 12098, 2, 9, 0.36363636363636365, 9, 0.391304347826087] + - ["RU", 1442034000000, 658, 3046, 1442041200000, 1442048400000, -324, 12098, 2, 10, 0.4090909090909091, 10, 0.43478260869565216] + - ["RU", 1442048400000, 1027, 4073, 1442080800000, 1442077200000, -324, 12098, 2, 11, 0.45454545454545453, 11, 0.4782608695652174] + - ["RU", 1442077200000, 1162, 5235, 1442034000000, 1442052000000, -324, 12098, 2, 12, 0.5, 12, 0.5217391304347826] + - ["RU", 1442052000000, 1214, 6449, 1442048400000, 1442073600000, -324, 12098, 2, 13, 0.5454545454545454, 13, 0.5652173913043478] + - ["RU", 1442073600000, 1618, 8067, 1442077200000, 1442066400000, -324, 12098, 2, 14, 0.5909090909090909, 14, 0.6086956521739131] + - ["RU", 1442066400000, 2047, 10114, 1442052000000, 1442019600000, -324, 12098, 2, 15, 0.6363636363636364, 15, 0.6521739130434783] + - ["RU", 1442019600000, 2214, 12328, 1442073600000, 1442044800000, -324, 12098, 2, 16, 0.6818181818181818, 16, 0.6956521739130435] + - ["RU", 1442044800000, 2564, 14892, 1442066400000, 1442088000000, -324, 12098, 3, 17, 0.7272727272727273, 17, 0.7391304347826086] + - ["RU", 1442088000000, 2596, 17488, 1442019600000, 1442091600000, -324, 12098, 3, 18, 0.7727272727272727, 18, 0.782608695652174] + - ["RU", 1442091600000, 3449, 20937, 1442044800000, 1442059200000, -324, 12098, 3, 19, 0.8181818181818182, 19, 0.8260869565217391] + - ["RU", 1442059200000, 3902, 24839, 1442088000000, 1442070000000, -324, 12098, 3, 20, 0.8636363636363636, 20, 0.8695652173913043] + - ["RU", 1442070000000, 4706, 29545, 1442091600000, 1442084400000, -324, 12098, 3, 21, 0.9090909090909091, 21, 0.9130434782608695] + - ["RU", 1442084400000, 6461, 36006, 1442059200000, 1442098800000, -324, 12098, 3, 22, 0.9545454545454546, 22, 0.9565217391304348] + - ["RU", 1442098800000, 12098, 48104, 1442070000000, null, -324, 12098, 3, 23, 1.0, 23, 1.0] + - ["SA", 1442037600000, -97, -97, null, 1442077200000, -97, 1276, 1, 1, 0.0, 1, 0.125] + - ["SA", 1442077200000, -50, -147, null, 1442059200000, -97, 1276, 1, 2, 0.14285714285714285, 2, 0.25] + - ["SA", 1442059200000, 0, -147, 1442037600000, 1442073600000, -97, 1276, 1, 3, 0.2857142857142857, 3, 0.375] + - ["SA", 1442073600000, 2, -145, 1442077200000, 1442055600000, -97, 1276, 2, 4, 0.42857142857142855, 4, 0.5] + - ["SA", 1442055600000, 11, -134, 1442059200000, 1442048400000, -97, 1276, 2, 5, 0.5714285714285714, 5, 0.625] + - ["SA", 1442048400000, 14, -120, 1442073600000, 1442084400000, -97, 1276, 2, 6, 0.7142857142857143, 6, 0.75] + - ["SA", 1442084400000, 458, 338, 1442055600000, 1442066400000, -97, 1276, 3, 7, 0.8571428571428571, 7, 0.875] + - ["SA", 1442066400000, 1276, 1614, 1442048400000, null, -97, 1276, 3, 8, 1.0, 8, 1.0] + - ["SE", 1442048400000, -145, -145, null, 1442055600000, -145, 1476, 1, 1, 0.0, 1, 0.06666666666666667] + - ["SE", 1442055600000, -5, -150, null, 1442091600000, -145, 1476, 1, 2, 0.07142857142857142, 2, 0.13333333333333333] + - ["SE", 1442091600000, -1, -151, 1442048400000, 1442098800000, -145, 1476, 1, 3, 0.14285714285714285, 3, 0.2] + - ["SE", 1442098800000, 0, -151, 1442055600000, 1442052000000, -145, 1476, 1, 4, 0.21428571428571427, 4, 0.26666666666666666] + - ["SE", 1442052000000, 1, -150, 1442091600000, 1442023200000, -145, 1476, 1, 5, 0.2857142857142857, 5, 0.3333333333333333] + - ["SE", 1442023200000, 3, -147, 1442098800000, 1442066400000, -145, 1476, 2, 6, 0.35714285714285715, 6, 0.4] + - ["SE", 1442066400000, 14, -133, 1442052000000, 1442030400000, -145, 1476, 2, 7, 0.42857142857142855, 7, 0.4666666666666667] + - ["SE", 1442030400000, 30, -103, 1442023200000, 1442084400000, -145, 1476, 2, 8, 0.5, 8, 0.5333333333333333] + - ["SE", 1442084400000, 37, -66, 1442066400000, 1442095200000, -145, 1476, 2, 9, 0.5714285714285714, 9, 0.6] + - ["SE", 1442095200000, 61, -5, 1442030400000, 1442070000000, -145, 1476, 2, 10, 0.6428571428571429, 10, 0.6666666666666666] + - ["SE", 1442070000000, 78, 73, 1442084400000, 1442080800000, -145, 1476, 3, 11, 0.7142857142857143, 11, 0.7333333333333333] + - ["SE", 1442080800000, 89, 162, 1442095200000, 1442041200000, -145, 1476, 3, 12, 0.7857142857142857, 12, 0.8] + - ["SE", 1442041200000, 91, 253, 1442070000000, 1442019600000, -145, 1476, 3, 13, 0.8571428571428571, 13, 0.8666666666666667] + - ["SE", 1442019600000, 109, 362, 1442080800000, 1442059200000, -145, 1476, 3, 14, 0.9285714285714286, 14, 0.9333333333333333] + - ["SE", 1442059200000, 1476, 1838, 1442041200000, null, -145, 1476, 3, 15, 1.0, 15, 1.0] + - ["SG", 1442066400000, 0, 0, null, 1442030400000, 0, 2758, 1, 1, 0.0, 1, 0.125] + - ["SG", 1442030400000, 1, 1, null, 1442037600000, 0, 2758, 1, 2, 0.14285714285714285, 2, 0.25] + - ["SG", 1442037600000, 3, 4, 1442066400000, 1442048400000, 0, 2758, 1, 3, 0.2857142857142857, 3, 0.375] + - ["SG", 1442048400000, 52, 56, 1442030400000, 1442041200000, 0, 2758, 2, 4, 0.42857142857142855, 4, 0.5] + - ["SG", 1442041200000, 59, 115, 1442037600000, 1442044800000, 0, 2758, 2, 5, 0.5714285714285714, 5, 0.625] + - ["SG", 1442044800000, 77, 192, 1442048400000, 1442062800000, 0, 2758, 2, 6, 0.7142857142857143, 6, 0.75] + - ["SG", 1442062800000, 388, 580, 1442041200000, 1442026800000, 0, 2758, 3, 7, 0.8571428571428571, 7, 0.875] + - ["SG", 1442026800000, 2758, 3338, 1442044800000, null, 0, 2758, 3, 8, 1.0, 8, 1.0] + - ["SI", 1442080800000, -45, -45, null, 1442091600000, -45, 9, 1, 1, 0.0, 1, 0.5] + - ["SI", 1442091600000, 9, -36, null, null, -45, 9, 2, 2, 1.0, 2, 1.0] + - ["SK", 1442084400000, -92, -92, null, 1442037600000, -92, 446, 1, 1, 0.0, 1, 0.16666666666666666] + - ["SK", 1442037600000, -1, -93, null, 1442062800000, -92, 446, 1, 2, 0.2, 2, 0.3333333333333333] + - ["SK", 1442062800000, 6, -87, 1442084400000, 1442098800000, -92, 446, 2, 3, 0.4, 3, 0.5] + - ["SK", 1442098800000, 7, -80, 1442037600000, 1442052000000, -92, 446, 2, 4, 0.6, 4, 0.6666666666666666] + - ["SK", 1442052000000, 13, -67, 1442062800000, 1442073600000, -92, 446, 3, 5, 0.8, 5, 0.8333333333333334] + - ["SK", 1442073600000, 446, 379, 1442098800000, null, -92, 446, 3, 6, 1.0, 6, 1.0] + - ["SV", 1442019600000, -1, -1, null, 1442088000000, -1, 106, 1, 1, 0.0, 1, 0.3333333333333333] + - ["SV", 1442088000000, 9, 8, null, 1442084400000, -1, 106, 2, 2, 0.5, 2, 0.6666666666666666] + - ["SV", 1442084400000, 106, 114, 1442019600000, null, -1, 106, 3, 3, 1.0, 3, 1.0] + - ["TH", 1442062800000, -46, -46, null, 1442066400000, -46, 110, 1, 1, 0.0, 1, 0.1111111111111111] + - ["TH", 1442066400000, -34, -80, null, 1442052000000, -46, 110, 1, 2, 0.125, 2, 0.2222222222222222] + - ["TH", 1442052000000, -22, -102, 1442062800000, 1442034000000, -46, 110, 1, 3, 0.25, 3, 0.3333333333333333] + - ["TH", 1442034000000, 0, -102, 1442066400000, 1442055600000, -46, 110, 2, 4, 0.375, 4, 0.6666666666666666] + - ["TH", 1442055600000, 0, -102, 1442052000000, 1442070000000, -46, 110, 2, 4, 0.375, 4, 0.6666666666666666] + - ["TH", 1442070000000, 0, -102, 1442034000000, 1442041200000, -46, 110, 2, 4, 0.375, 4, 0.6666666666666666] + - ["TH", 1442041200000, 3, -99, 1442055600000, 1442084400000, -46, 110, 3, 7, 0.75, 5, 0.7777777777777778] + - ["TH", 1442084400000, 13, -86, 1442070000000, 1442044800000, -46, 110, 3, 8, 0.875, 6, 0.8888888888888888] + - ["TH", 1442044800000, 110, 24, 1442041200000, null, -46, 110, 3, 9, 1.0, 7, 1.0] + - ["TJ", 1442048400000, 1471, 1471, null, null, 1471, 1471, 1, 1, 0.0, 1, 1.0] + - ["TN", 1442098800000, -9, -9, null, null, -9, -9, 1, 1, 0.0, 1, 1.0] + - ["TR", 1442095200000, -29, -29, null, 1442080800000, -29, 3048, 1, 1, 0.0, 1, 0.06666666666666667] + - ["TR", 1442080800000, -1, -30, null, 1442041200000, -29, 3048, 1, 2, 0.07142857142857142, 2, 0.13333333333333333] + - ["TR", 1442041200000, 1, -29, 1442095200000, 1442044800000, -29, 3048, 1, 3, 0.14285714285714285, 3, 0.2] + - ["TR", 1442044800000, 41, 12, 1442080800000, 1442052000000, -29, 3048, 1, 4, 0.21428571428571427, 4, 0.3333333333333333] + - ["TR", 1442052000000, 41, 53, 1442041200000, 1442066400000, -29, 3048, 1, 4, 0.21428571428571427, 4, 0.3333333333333333] + - ["TR", 1442066400000, 85, 138, 1442044800000, 1442048400000, -29, 3048, 2, 6, 0.35714285714285715, 5, 0.4] + - ["TR", 1442048400000, 88, 226, 1442052000000, 1442077200000, -29, 3048, 2, 7, 0.42857142857142855, 6, 0.4666666666666667] + - ["TR", 1442077200000, 89, 315, 1442066400000, 1442084400000, -29, 3048, 2, 8, 0.5, 7, 0.5333333333333333] + - ["TR", 1442084400000, 170, 485, 1442048400000, 1442070000000, -29, 3048, 2, 9, 0.5714285714285714, 8, 0.6] + - ["TR", 1442070000000, 236, 721, 1442077200000, 1442055600000, -29, 3048, 2, 10, 0.6428571428571429, 9, 0.6666666666666666] + - ["TR", 1442055600000, 299, 1020, 1442084400000, 1442023200000, -29, 3048, 3, 11, 0.7142857142857143, 10, 0.7333333333333333] + - ["TR", 1442023200000, 306, 1326, 1442070000000, 1442062800000, -29, 3048, 3, 12, 0.7857142857142857, 11, 0.8] + - ["TR", 1442062800000, 315, 1641, 1442055600000, 1442088000000, -29, 3048, 3, 13, 0.8571428571428571, 12, 0.8666666666666667] + - ["TR", 1442088000000, 2389, 4030, 1442023200000, 1442091600000, -29, 3048, 3, 14, 0.9285714285714286, 13, 0.9333333333333333] + - ["TR", 1442091600000, 3048, 7078, 1442062800000, null, -29, 3048, 3, 15, 1.0, 14, 1.0] + - ["TT", 1442088000000, 9, 9, null, null, 9, 9, 1, 1, 0.0, 1, 1.0] + - ["TW", 1442062800000, -272, -272, null, 1442059200000, -272, 772, 1, 1, 0.0, 1, 0.045454545454545456] + - ["TW", 1442059200000, -157, -429, null, 1442095200000, -272, 772, 1, 2, 0.047619047619047616, 2, 0.09090909090909091] + - ["TW", 1442095200000, -77, -506, 1442062800000, 1442098800000, -272, 772, 1, 3, 0.09523809523809523, 3, 0.13636363636363635] + - ["TW", 1442098800000, -60, -566, 1442059200000, 1442019600000, -272, 772, 1, 4, 0.14285714285714285, 4, 0.18181818181818182] + - ["TW", 1442019600000, 0, -566, 1442095200000, 1442030400000, -272, 772, 1, 5, 0.19047619047619047, 5, 0.3181818181818182] + - ["TW", 1442030400000, 0, -566, 1442098800000, 1442084400000, -272, 772, 1, 5, 0.19047619047619047, 5, 0.3181818181818182] + - ["TW", 1442084400000, 0, -566, 1442019600000, 1442044800000, -272, 772, 1, 5, 0.19047619047619047, 5, 0.3181818181818182] + - ["TW", 1442044800000, 24, -542, 1442030400000, 1442052000000, -272, 772, 1, 8, 0.3333333333333333, 6, 0.45454545454545453] + - ["TW", 1442052000000, 24, -518, 1442084400000, 1442080800000, -272, 772, 2, 8, 0.3333333333333333, 6, 0.45454545454545453] + - ["TW", 1442080800000, 24, -494, 1442044800000, 1442055600000, -272, 772, 2, 8, 0.3333333333333333, 6, 0.45454545454545453] + - ["TW", 1442055600000, 48, -446, 1442052000000, 1442048400000, -272, 772, 2, 11, 0.47619047619047616, 7, 0.5] + - ["TW", 1442048400000, 75, -371, 1442080800000, 1442016000000, -272, 772, 2, 12, 0.5238095238095238, 8, 0.5454545454545454] + - ["TW", 1442016000000, 92, -279, 1442055600000, 1442023200000, -272, 772, 2, 13, 0.5714285714285714, 9, 0.5909090909090909] + - ["TW", 1442023200000, 97, -182, 1442048400000, 1442034000000, -272, 772, 2, 14, 0.6190476190476191, 10, 0.6363636363636364] + - ["TW", 1442034000000, 143, -39, 1442016000000, 1442037600000, -272, 772, 2, 15, 0.6666666666666666, 11, 0.6818181818181818] + - ["TW", 1442037600000, 266, 227, 1442023200000, 1442041200000, -272, 772, 3, 16, 0.7142857142857143, 12, 0.7272727272727273] + - ["TW", 1442041200000, 366, 593, 1442034000000, 1442070000000, -272, 772, 3, 17, 0.7619047619047619, 13, 0.7727272727272727] + - ["TW", 1442070000000, 485, 1078, 1442037600000, 1442077200000, -272, 772, 3, 18, 0.8095238095238095, 14, 0.8181818181818182] + - ["TW", 1442077200000, 502, 1580, 1442041200000, 1442066400000, -272, 772, 3, 19, 0.8571428571428571, 15, 0.8636363636363636] + - ["TW", 1442066400000, 624, 2204, 1442070000000, 1442026800000, -272, 772, 3, 20, 0.9047619047619048, 16, 0.9090909090909091] + - ["TW", 1442026800000, 680, 2884, 1442077200000, 1442073600000, -272, 772, 3, 21, 0.9523809523809523, 17, 0.9545454545454546] + - ["TW", 1442073600000, 772, 3656, 1442066400000, null, -272, 772, 3, 22, 1.0, 18, 1.0] + - ["UA", 1442091600000, -388, -388, null, 1442077200000, -388, 14202, 1, 1, 0.0, 1, 0.05263157894736842] + - ["UA", 1442077200000, -181, -569, null, 1442095200000, -388, 14202, 1, 2, 0.05555555555555555, 2, 0.10526315789473684] + - ["UA", 1442095200000, -30, -599, 1442091600000, 1442088000000, -388, 14202, 1, 3, 0.1111111111111111, 3, 0.15789473684210525] + - ["UA", 1442088000000, -21, -620, 1442077200000, 1442059200000, -388, 14202, 1, 4, 0.16666666666666666, 4, 0.21052631578947367] + - ["UA", 1442059200000, -2, -622, 1442095200000, 1442037600000, -388, 14202, 1, 5, 0.2222222222222222, 5, 0.2631578947368421] + - ["UA", 1442037600000, -1, -623, 1442088000000, 1442080800000, -388, 14202, 1, 6, 0.2777777777777778, 6, 0.3684210526315789] + - ["UA", 1442080800000, -1, -624, 1442059200000, 1442048400000, -388, 14202, 1, 6, 0.2777777777777778, 6, 0.3684210526315789] + - ["UA", 1442048400000, 2, -622, 1442037600000, 1442084400000, -388, 14202, 2, 8, 0.3888888888888889, 7, 0.42105263157894735] + - ["UA", 1442084400000, 5, -617, 1442080800000, 1442098800000, -388, 14202, 2, 9, 0.4444444444444444, 8, 0.47368421052631576] + - ["UA", 1442098800000, 38, -579, 1442048400000, 1442041200000, -388, 14202, 2, 10, 0.5, 9, 0.5263157894736842] + - ["UA", 1442041200000, 74, -505, 1442084400000, 1442044800000, -388, 14202, 2, 11, 0.5555555555555556, 10, 0.5789473684210527] + - ["UA", 1442044800000, 280, -225, 1442098800000, 1442066400000, -388, 14202, 2, 12, 0.6111111111111112, 11, 0.631578947368421] + - ["UA", 1442066400000, 296, 71, 1442041200000, 1442052000000, -388, 14202, 2, 13, 0.6666666666666666, 12, 0.6842105263157895] + - ["UA", 1442052000000, 410, 481, 1442044800000, 1442062800000, -388, 14202, 3, 14, 0.7222222222222222, 13, 0.7368421052631579] + - ["UA", 1442062800000, 773, 1254, 1442066400000, 1442070000000, -388, 14202, 3, 15, 0.7777777777777778, 14, 0.7894736842105263] + - ["UA", 1442070000000, 1733, 2987, 1442052000000, 1442034000000, -388, 14202, 3, 16, 0.8333333333333334, 15, 0.8421052631578947] + - ["UA", 1442034000000, 3468, 6455, 1442062800000, 1442073600000, -388, 14202, 3, 17, 0.8888888888888888, 16, 0.8947368421052632] + - ["UA", 1442073600000, 4241, 10696, 1442070000000, 1442055600000, -388, 14202, 3, 18, 0.9444444444444444, 17, 0.9473684210526315] + - ["UA", 1442055600000, 14202, 24898, 1442034000000, null, -388, 14202, 3, 19, 1.0, 18, 1.0] + - ["UG", 1442070000000, 1, 1, null, null, 1, 1, 1, 1, 0.0, 1, 1.0] + - ["US", 1442048400000, -466, -466, null, 1442052000000, -466, 4001, 1, 1, 0.0, 1, 0.041666666666666664] + - ["US", 1442052000000, -2, -468, null, 1442016000000, -466, 4001, 1, 2, 0.043478260869565216, 2, 0.08333333333333333] + - ["US", 1442016000000, 0, -468, 1442048400000, 1442059200000, -466, 4001, 1, 3, 0.08695652173913043, 3, 0.125] + - ["US", 1442059200000, 11, -457, 1442052000000, 1442062800000, -466, 4001, 1, 4, 0.13043478260869565, 4, 0.16666666666666666] + - ["US", 1442062800000, 47, -410, 1442016000000, 1442044800000, -466, 4001, 1, 5, 0.17391304347826086, 5, 0.20833333333333334] + - ["US", 1442044800000, 139, -271, 1442059200000, 1442055600000, -466, 4001, 1, 6, 0.21739130434782608, 6, 0.25] + - ["US", 1442055600000, 156, -115, 1442062800000, 1442095200000, -466, 4001, 1, 7, 0.2608695652173913, 7, 0.2916666666666667] + - ["US", 1442095200000, 416, 301, 1442044800000, 1442066400000, -466, 4001, 1, 8, 0.30434782608695654, 8, 0.3333333333333333] + - ["US", 1442066400000, 772, 1073, 1442055600000, 1442019600000, -466, 4001, 2, 9, 0.34782608695652173, 9, 0.375] + - ["US", 1442019600000, 1043, 2116, 1442095200000, 1442073600000, -466, 4001, 2, 10, 0.391304347826087, 10, 0.4166666666666667] + - ["US", 1442073600000, 1100, 3216, 1442066400000, 1442026800000, -466, 4001, 2, 11, 0.43478260869565216, 11, 0.4583333333333333] + - ["US", 1442026800000, 1512, 4728, 1442019600000, 1442088000000, -466, 4001, 2, 12, 0.4782608695652174, 12, 0.5] + - ["US", 1442088000000, 1691, 6419, 1442073600000, 1442041200000, -466, 4001, 2, 13, 0.5217391304347826, 13, 0.5416666666666666] + - ["US", 1442041200000, 1999, 8418, 1442026800000, 1442030400000, -466, 4001, 2, 14, 0.5652173913043478, 14, 0.5833333333333334] + - ["US", 1442030400000, 2023, 10441, 1442088000000, 1442077200000, -466, 4001, 2, 15, 0.6086956521739131, 15, 0.625] + - ["US", 1442077200000, 2168, 12609, 1442041200000, 1442091600000, -466, 4001, 2, 16, 0.6521739130434783, 16, 0.6666666666666666] + - ["US", 1442091600000, 2502, 15111, 1442030400000, 1442084400000, -466, 4001, 3, 17, 0.6956521739130435, 17, 0.7083333333333334] + - ["US", 1442084400000, 2523, 17634, 1442077200000, 1442023200000, -466, 4001, 3, 18, 0.7391304347826086, 18, 0.75] + - ["US", 1442023200000, 2844, 20478, 1442091600000, 1442070000000, -466, 4001, 3, 19, 0.782608695652174, 19, 0.7916666666666666] + - ["US", 1442070000000, 3505, 23983, 1442084400000, 1442098800000, -466, 4001, 3, 20, 0.8260869565217391, 20, 0.8333333333333334] + - ["US", 1442098800000, 3575, 27558, 1442023200000, 1442034000000, -466, 4001, 3, 21, 0.8695652173913043, 21, 0.875] + - ["US", 1442034000000, 3648, 31206, 1442070000000, 1442037600000, -466, 4001, 3, 22, 0.9130434782608695, 22, 0.9166666666666666] + - ["US", 1442037600000, 3675, 34881, 1442098800000, 1442080800000, -466, 4001, 3, 23, 0.9565217391304348, 23, 0.9583333333333334] + - ["US", 1442080800000, 4001, 38882, 1442034000000, null, -466, 4001, 3, 24, 1.0, 24, 1.0] + - ["UY", 1442073600000, -42, -42, null, 1442037600000, -42, 517, 1, 1, 0.0, 1, 0.14285714285714285] + - ["UY", 1442037600000, 1, -41, null, 1442077200000, -42, 517, 1, 2, 0.16666666666666666, 2, 0.2857142857142857] + - ["UY", 1442077200000, 23, -18, 1442073600000, 1442026800000, -42, 517, 1, 3, 0.3333333333333333, 3, 0.42857142857142855] + - ["UY", 1442026800000, 76, 58, 1442037600000, 1442019600000, -42, 517, 2, 4, 0.5, 4, 0.5714285714285714] + - ["UY", 1442019600000, 77, 135, 1442077200000, 1442070000000, -42, 517, 2, 5, 0.6666666666666666, 5, 0.7142857142857143] + - ["UY", 1442070000000, 284, 419, 1442026800000, 1442023200000, -42, 517, 3, 6, 0.8333333333333334, 6, 0.8571428571428571] + - ["UY", 1442023200000, 517, 936, 1442019600000, null, -42, 517, 3, 7, 1.0, 7, 1.0] + - ["UZ", 1442044800000, 1369, 1369, null, null, 1369, 1369, 1, 1, 0.0, 1, 1.0] + - ["VE", 1442026800000, -17, -17, null, 1442034000000, -17, 420, 1, 1, 0.0, 1, 0.1] + - ["VE", 1442034000000, -2, -19, null, 1442098800000, -17, 420, 1, 2, 0.1111111111111111, 2, 0.2] + - ["VE", 1442098800000, 9, -10, 1442026800000, 1442066400000, -17, 420, 1, 3, 0.2222222222222222, 3, 0.3] + - ["VE", 1442066400000, 18, 8, 1442034000000, 1442095200000, -17, 420, 1, 4, 0.3333333333333333, 4, 0.4] + - ["VE", 1442095200000, 35, 43, 1442098800000, 1442030400000, -17, 420, 2, 5, 0.4444444444444444, 5, 0.5] + - ["VE", 1442030400000, 51, 94, 1442066400000, 1442084400000, -17, 420, 2, 6, 0.5555555555555556, 6, 0.6] + - ["VE", 1442084400000, 60, 154, 1442095200000, 1442023200000, -17, 420, 2, 7, 0.6666666666666666, 7, 0.7] + - ["VE", 1442023200000, 115, 269, 1442030400000, 1442077200000, -17, 420, 3, 8, 0.7777777777777778, 8, 0.8] + - ["VE", 1442077200000, 412, 681, 1442084400000, 1442070000000, -17, 420, 3, 9, 0.8888888888888888, 9, 0.9] + - ["VE", 1442070000000, 420, 1101, 1442023200000, null, -17, 420, 3, 10, 1.0, 10, 1.0] + - ["VG", 1442062800000, -238, -238, null, null, -238, -238, 1, 1, 0.0, 1, 1.0] + - ["VN", 1442034000000, -29, -29, null, 1442048400000, -29, 811, 1, 1, 0.0, 1, 0.07692307692307693] + - ["VN", 1442048400000, -15, -44, null, 1442037600000, -29, 811, 1, 2, 0.08333333333333333, 2, 0.15384615384615385] + - ["VN", 1442037600000, -11, -55, 1442034000000, 1442084400000, -29, 811, 1, 3, 0.16666666666666666, 3, 0.23076923076923078] + - ["VN", 1442084400000, -10, -65, 1442048400000, 1442023200000, -29, 811, 1, 4, 0.25, 4, 0.3076923076923077] + - ["VN", 1442023200000, -9, -74, 1442037600000, 1442041200000, -29, 811, 1, 5, 0.3333333333333333, 5, 0.38461538461538464] + - ["VN", 1442041200000, 0, -74, 1442084400000, 1442059200000, -29, 811, 2, 6, 0.4166666666666667, 6, 0.46153846153846156] + - ["VN", 1442059200000, 8, -66, 1442023200000, 1442055600000, -29, 811, 2, 7, 0.5, 7, 0.5384615384615384] + - ["VN", 1442055600000, 37, -29, 1442041200000, 1442026800000, -29, 811, 2, 8, 0.5833333333333334, 8, 0.6153846153846154] + - ["VN", 1442026800000, 63, 34, 1442059200000, 1442052000000, -29, 811, 2, 9, 0.6666666666666666, 9, 0.6923076923076923] + - ["VN", 1442052000000, 90, 124, 1442055600000, 1442062800000, -29, 811, 3, 10, 0.75, 10, 0.7692307692307693] + - ["VN", 1442062800000, 146, 270, 1442026800000, 1442070000000, -29, 811, 3, 11, 0.8333333333333334, 11, 0.8461538461538461] + - ["VN", 1442070000000, 479, 749, 1442052000000, 1442066400000, -29, 811, 3, 12, 0.9166666666666666, 12, 0.9230769230769231] + - ["VN", 1442066400000, 811, 1560, 1442062800000, null, -29, 811, 3, 13, 1.0, 13, 1.0] + - ["ZA", 1442034000000, -3, -3, null, 1442070000000, -3, 79, 1, 1, 0.0, 1, 0.2] + - ["ZA", 1442070000000, 0, -3, null, 1442091600000, -3, 79, 1, 2, 0.25, 2, 0.4] + - ["ZA", 1442091600000, 1, -2, 1442034000000, 1442059200000, -3, 79, 2, 3, 0.5, 3, 0.6] + - ["ZA", 1442059200000, 50, 48, 1442070000000, 1442048400000, -3, 79, 2, 4, 0.75, 4, 0.8] + - ["ZA", 1442048400000, 79, 127, 1442091600000, null, -3, 79, 3, 5, 1.0, 5, 1.0] + - ["ZM", 1442041200000, 133, 133, null, null, 133, 133, 1, 1, 0.0, 1, 1.0] + - ["ZW", 1442044800000, 0, 0, null, 1442048400000, 0, 254, 1, 1, 0.0, 1, 0.5] + - ["ZW", 1442048400000, 254, 254, null, null, 0, 254, 2, 2, 1.0, 2, 1.0] \ No newline at end of file diff --git a/sql/src/test/resources/calcite/tests/window/wikipediaSimplePartition.sqlTest b/sql/src/test/resources/calcite/tests/window/wikipediaSimplePartition.sqlTest new file mode 100644 index 000000000000..418959469535 --- /dev/null +++ b/sql/src/test/resources/calcite/tests/window/wikipediaSimplePartition.sqlTest @@ -0,0 +1,1019 @@ +type: "operatorValidation" + +sql: " + SELECT + countryIsoCode, + FLOOR(__time TO HOUR) t, + SUM(delta) delta, + SUM(SUM(delta)) OVER (PARTITION BY countryIsoCode) totalDelta, + LAG(FLOOR(__time TO HOUR), 2) OVER (PARTITION BY countryIsoCode) laggardTime, + LEAD(FLOOR(__time TO HOUR), 1) OVER (PARTITION BY countryIsoCode) leadTime, + FIRST_VALUE(SUM(delta)) OVER (PARTITION BY countryIsoCode) AS firstDelay, + LAST_VALUE(SUM(delta)) OVER (PARTITION BY countryIsoCode) AS lastDelay, + NTILE(3) OVER (PARTITION BY countryIsoCode) AS delayNTile + FROM wikipedia + GROUP BY 1, 2" + +expectedOperators: + - { type: "naivePartition", partitionColumns: ["d0"] } + - type: "window" + processor: + type: "composing" + processors: + - { "type":"offset", "inputColumn":"d1", "outputColumn":"w1", "offset":-2 } + - { "type":"offset", "inputColumn":"d1", "outputColumn":"w2", "offset":1 } + - { "type":"first", "inputColumn":"a0", "outputColumn":"w3" } + - { "type":"last", "inputColumn":"a0", "outputColumn":"w4" } + - { "type": "percentile", "outputColumn": "w5", "numBuckets": 3 } + - type: "aggregate" + aggregations: + - { "type":"longSum", "name":"w0", "fieldName":"a0" } + +expectedResults: + - ["", 1442016000000, 29873, 8414700, null, 1442019600000, 29873, 276159, 1] + - ["", 1442019600000, 173892, 8414700, null, 1442023200000, 29873, 276159, 1] + - ["", 1442023200000, 399636, 8414700, 1442016000000, 1442026800000, 29873, 276159, 1] + - ["", 1442026800000, 252626, 8414700, 1442019600000, 1442030400000, 29873, 276159, 1] + - ["", 1442030400000, 166672, 8414700, 1442023200000, 1442034000000, 29873, 276159, 1] + - ["", 1442034000000, 330957, 8414700, 1442026800000, 1442037600000, 29873, 276159, 1] + - ["", 1442037600000, 200605, 8414700, 1442030400000, 1442041200000, 29873, 276159, 1] + - ["", 1442041200000, 543450, 8414700, 1442034000000, 1442044800000, 29873, 276159, 1] + - ["", 1442044800000, 316002, 8414700, 1442037600000, 1442048400000, 29873, 276159, 2] + - ["", 1442048400000, 308316, 8414700, 1442041200000, 1442052000000, 29873, 276159, 2] + - ["", 1442052000000, 787370, 8414700, 1442044800000, 1442055600000, 29873, 276159, 2] + - ["", 1442055600000, 283958, 8414700, 1442048400000, 1442059200000, 29873, 276159, 2] + - ["", 1442059200000, 459297, 8414700, 1442052000000, 1442062800000, 29873, 276159, 2] + - ["", 1442062800000, 389465, 8414700, 1442055600000, 1442066400000, 29873, 276159, 2] + - ["", 1442066400000, 351584, 8414700, 1442059200000, 1442070000000, 29873, 276159, 2] + - ["", 1442070000000, 358515, 8414700, 1442062800000, 1442073600000, 29873, 276159, 2] + - ["", 1442073600000, 375394, 8414700, 1442066400000, 1442077200000, 29873, 276159, 3] + - ["", 1442077200000, 392483, 8414700, 1442070000000, 1442080800000, 29873, 276159, 3] + - ["", 1442080800000, 453077, 8414700, 1442073600000, 1442084400000, 29873, 276159, 3] + - ["", 1442084400000, 372569, 8414700, 1442077200000, 1442088000000, 29873, 276159, 3] + - ["", 1442088000000, 303872, 8414700, 1442080800000, 1442091600000, 29873, 276159, 3] + - ["", 1442091600000, 514427, 8414700, 1442084400000, 1442095200000, 29873, 276159, 3] + - ["", 1442095200000, 374501, 8414700, 1442088000000, 1442098800000, 29873, 276159, 3] + - ["", 1442098800000, 276159, 8414700, 1442091600000, null, 29873, 276159, 3] + - ["AE", 1442030400000, 118, 6547, null, 1442044800000, 118, 42, 1] + - ["AE", 1442044800000, -7, 6547, null, 1442048400000, 118, 42, 1] + - ["AE", 1442048400000, 39, 6547, 1442030400000, 1442052000000, 118, 42, 1] + - ["AE", 1442052000000, -3, 6547, 1442044800000, 1442059200000, 118, 42, 2] + - ["AE", 1442059200000, -11, 6547, 1442048400000, 1442070000000, 118, 42, 2] + - ["AE", 1442070000000, 46, 6547, 1442052000000, 1442077200000, 118, 42, 2] + - ["AE", 1442077200000, 6323, 6547, 1442059200000, 1442080800000, 118, 42, 3] + - ["AE", 1442080800000, 42, 6547, 1442070000000, null, 118, 42, 3] + - ["AL", 1442077200000, 26, 80, null, 1442091600000, 26, 54, 1] + - ["AL", 1442091600000, 54, 80, null, null, 26, 54, 2] + - ["AO", 1442041200000, -26, 740, null, 1442052000000, -26, 722, 1] + - ["AO", 1442052000000, -18, 740, null, 1442088000000, -26, 722, 1] + - ["AO", 1442088000000, 62, 740, 1442041200000, 1442098800000, -26, 722, 2] + - ["AO", 1442098800000, 722, 740, 1442052000000, null, -26, 722, 3] + - ["AR", 1442019600000, 1, 4450, null, 1442023200000, 1, 64, 1] + - ["AR", 1442023200000, 2514, 4450, null, 1442026800000, 1, 64, 1] + - ["AR", 1442026800000, 644, 4450, 1442019600000, 1442030400000, 1, 64, 1] + - ["AR", 1442030400000, -3, 4450, 1442023200000, 1442034000000, 1, 64, 1] + - ["AR", 1442034000000, 212, 4450, 1442026800000, 1442037600000, 1, 64, 1] + - ["AR", 1442037600000, 81, 4450, 1442030400000, 1442055600000, 1, 64, 1] + - ["AR", 1442055600000, -54, 4450, 1442034000000, 1442059200000, 1, 64, 2] + - ["AR", 1442059200000, 210, 4450, 1442037600000, 1442062800000, 1, 64, 2] + - ["AR", 1442062800000, 29, 4450, 1442055600000, 1442066400000, 1, 64, 2] + - ["AR", 1442066400000, 0, 4450, 1442059200000, 1442070000000, 1, 64, 2] + - ["AR", 1442070000000, 377, 4450, 1442062800000, 1442077200000, 1, 64, 2] + - ["AR", 1442077200000, -591, 4450, 1442066400000, 1442080800000, 1, 64, 2] + - ["AR", 1442080800000, 1, 4450, 1442070000000, 1442084400000, 1, 64, 3] + - ["AR", 1442084400000, -5, 4450, 1442077200000, 1442091600000, 1, 64, 3] + - ["AR", 1442091600000, 340, 4450, 1442080800000, 1442095200000, 1, 64, 3] + - ["AR", 1442095200000, 630, 4450, 1442084400000, 1442098800000, 1, 64, 3] + - ["AR", 1442098800000, 64, 4450, 1442091600000, null, 1, 64, 3] + - ["AT", 1442052000000, 4793, 12047, null, 1442062800000, 4793, 89, 1] + - ["AT", 1442062800000, -155, 12047, null, 1442066400000, 4793, 89, 1] + - ["AT", 1442066400000, 0, 12047, 1442052000000, 1442070000000, 4793, 89, 1] + - ["AT", 1442070000000, 272, 12047, 1442062800000, 1442084400000, 4793, 89, 2] + - ["AT", 1442084400000, -2, 12047, 1442066400000, 1442088000000, 4793, 89, 2] + - ["AT", 1442088000000, 7050, 12047, 1442070000000, 1442091600000, 4793, 89, 3] + - ["AT", 1442091600000, 89, 12047, 1442084400000, null, 4793, 89, 3] + - ["AU", 1442016000000, 0, 2700, null, 1442019600000, 0, 518, 1] + - ["AU", 1442019600000, 253, 2700, null, 1442023200000, 0, 518, 1] + - ["AU", 1442023200000, 52, 2700, 1442016000000, 1442026800000, 0, 518, 1] + - ["AU", 1442026800000, 188, 2700, 1442019600000, 1442030400000, 0, 518, 1] + - ["AU", 1442030400000, -377, 2700, 1442023200000, 1442034000000, 0, 518, 1] + - ["AU", 1442034000000, 283, 2700, 1442026800000, 1442037600000, 0, 518, 1] + - ["AU", 1442037600000, 3, 2700, 1442030400000, 1442041200000, 0, 518, 1] + - ["AU", 1442041200000, 194, 2700, 1442034000000, 1442044800000, 0, 518, 2] + - ["AU", 1442044800000, 373, 2700, 1442037600000, 1442048400000, 0, 518, 2] + - ["AU", 1442048400000, 135, 2700, 1442041200000, 1442052000000, 0, 518, 2] + - ["AU", 1442052000000, -643, 2700, 1442044800000, 1442055600000, 0, 518, 2] + - ["AU", 1442055600000, 182, 2700, 1442048400000, 1442059200000, 0, 518, 2] + - ["AU", 1442059200000, 38, 2700, 1442052000000, 1442066400000, 0, 518, 2] + - ["AU", 1442066400000, -21, 2700, 1442055600000, 1442070000000, 0, 518, 3] + - ["AU", 1442070000000, -12, 2700, 1442059200000, 1442077200000, 0, 518, 3] + - ["AU", 1442077200000, 1, 2700, 1442066400000, 1442091600000, 0, 518, 3] + - ["AU", 1442091600000, 1138, 2700, 1442070000000, 1442095200000, 0, 518, 3] + - ["AU", 1442095200000, 395, 2700, 1442077200000, 1442098800000, 0, 518, 3] + - ["AU", 1442098800000, 518, 2700, 1442091600000, null, 0, 518, 3] + - ["BA", 1442048400000, -13, -178, null, 1442052000000, -13, -1, 1] + - ["BA", 1442052000000, 38, -178, null, 1442055600000, -13, -1, 1] + - ["BA", 1442055600000, -202, -178, 1442048400000, 1442084400000, -13, -1, 2] + - ["BA", 1442084400000, -1, -178, 1442052000000, null, -13, -1, 3] + - ["BD", 1442019600000, 0, 1106, null, 1442041200000, 0, -2, 1] + - ["BD", 1442041200000, 854, 1106, null, 1442066400000, 0, -2, 1] + - ["BD", 1442066400000, 76, 1106, 1442019600000, 1442073600000, 0, -2, 2] + - ["BD", 1442073600000, 103, 1106, 1442041200000, 1442077200000, 0, -2, 2] + - ["BD", 1442077200000, 75, 1106, 1442066400000, 1442091600000, 0, -2, 3] + - ["BD", 1442091600000, -2, 1106, 1442073600000, null, 0, -2, 3] + - ["BE", 1442030400000, -103, 795, null, 1442048400000, -103, 9, 1] + - ["BE", 1442048400000, 59, 795, null, 1442052000000, -103, 9, 1] + - ["BE", 1442052000000, -1, 795, 1442030400000, 1442055600000, -103, 9, 1] + - ["BE", 1442055600000, 233, 795, 1442048400000, 1442062800000, -103, 9, 1] + - ["BE", 1442062800000, 91, 795, 1442052000000, 1442066400000, -103, 9, 2] + - ["BE", 1442066400000, 136, 795, 1442055600000, 1442073600000, -103, 9, 2] + - ["BE", 1442073600000, 19, 795, 1442062800000, 1442080800000, -103, 9, 2] + - ["BE", 1442080800000, 1, 795, 1442066400000, 1442084400000, -103, 9, 2] + - ["BE", 1442084400000, 183, 795, 1442073600000, 1442088000000, -103, 9, 3] + - ["BE", 1442088000000, 67, 795, 1442080800000, 1442091600000, -103, 9, 3] + - ["BE", 1442091600000, 101, 795, 1442084400000, 1442098800000, -103, 9, 3] + - ["BE", 1442098800000, 9, 795, 1442088000000, null, -103, 9, 3] + - ["BG", 1442041200000, 9, 19592, null, 1442052000000, 9, 401, 1] + - ["BG", 1442052000000, 18936, 19592, null, 1442059200000, 9, 401, 1] + - ["BG", 1442059200000, 191, 19592, 1442041200000, 1442070000000, 9, 401, 2] + - ["BG", 1442070000000, 55, 19592, 1442052000000, 1442084400000, 9, 401, 2] + - ["BG", 1442084400000, 401, 19592, 1442059200000, null, 9, 401, 3] + - ["BH", 1442052000000, 44, 44, null, null, 44, 44, 1] + - ["BO", 1442080800000, 4, 4, null, 1442088000000, 4, -4, 1] + - ["BO", 1442088000000, 4, 4, null, 1442095200000, 4, -4, 2] + - ["BO", 1442095200000, -4, 4, 1442080800000, null, 4, -4, 3] + - ["BR", 1442016000000, -248, 8550, null, 1442019600000, -248, -645, 1] + - ["BR", 1442019600000, 372, 8550, null, 1442023200000, -248, -645, 1] + - ["BR", 1442023200000, 879, 8550, 1442016000000, 1442026800000, -248, -645, 1] + - ["BR", 1442026800000, 51, 8550, 1442019600000, 1442030400000, -248, -645, 1] + - ["BR", 1442030400000, 30, 8550, 1442023200000, 1442034000000, -248, -645, 1] + - ["BR", 1442034000000, 21, 8550, 1442026800000, 1442037600000, -248, -645, 1] + - ["BR", 1442037600000, 267, 8550, 1442030400000, 1442041200000, -248, -645, 1] + - ["BR", 1442041200000, 3, 8550, 1442034000000, 1442044800000, -248, -645, 1] + - ["BR", 1442044800000, 71, 8550, 1442037600000, 1442052000000, -248, -645, 2] + - ["BR", 1442052000000, 232, 8550, 1442041200000, 1442055600000, -248, -645, 2] + - ["BR", 1442055600000, 242, 8550, 1442044800000, 1442059200000, -248, -645, 2] + - ["BR", 1442059200000, 73, 8550, 1442052000000, 1442062800000, -248, -645, 2] + - ["BR", 1442062800000, 93, 8550, 1442055600000, 1442066400000, -248, -645, 2] + - ["BR", 1442066400000, 1034, 8550, 1442059200000, 1442070000000, -248, -645, 2] + - ["BR", 1442070000000, 536, 8550, 1442062800000, 1442073600000, -248, -645, 2] + - ["BR", 1442073600000, 2087, 8550, 1442066400000, 1442077200000, -248, -645, 2] + - ["BR", 1442077200000, 2253, 8550, 1442070000000, 1442080800000, -248, -645, 3] + - ["BR", 1442080800000, -267, 8550, 1442073600000, 1442084400000, -248, -645, 3] + - ["BR", 1442084400000, 492, 8550, 1442077200000, 1442088000000, -248, -645, 3] + - ["BR", 1442088000000, 215, 8550, 1442080800000, 1442091600000, -248, -645, 3] + - ["BR", 1442091600000, 11, 8550, 1442084400000, 1442095200000, -248, -645, 3] + - ["BR", 1442095200000, 748, 8550, 1442088000000, 1442098800000, -248, -645, 3] + - ["BR", 1442098800000, -645, 8550, 1442091600000, null, -248, -645, 3] + - ["BY", 1442055600000, 1, 2153, null, 1442059200000, 1, 33, 1] + - ["BY", 1442059200000, 1464, 2153, null, 1442073600000, 1, 33, 1] + - ["BY", 1442073600000, 596, 2153, 1442055600000, 1442077200000, 1, 33, 1] + - ["BY", 1442077200000, 30, 2153, 1442059200000, 1442080800000, 1, 33, 2] + - ["BY", 1442080800000, 28, 2153, 1442073600000, 1442084400000, 1, 33, 2] + - ["BY", 1442084400000, 1, 2153, 1442077200000, 1442088000000, 1, 33, 3] + - ["BY", 1442088000000, 33, 2153, 1442080800000, null, 1, 33, 3] + - ["CA", 1442016000000, -371, 8544, null, 1442019600000, -371, 164, 1] + - ["CA", 1442019600000, 2184, 8544, null, 1442023200000, -371, 164, 1] + - ["CA", 1442023200000, 286, 8544, 1442016000000, 1442026800000, -371, 164, 1] + - ["CA", 1442026800000, 2216, 8544, 1442019600000, 1442030400000, -371, 164, 1] + - ["CA", 1442030400000, -47, 8544, 1442023200000, 1442034000000, -371, 164, 1] + - ["CA", 1442034000000, 178, 8544, 1442026800000, 1442037600000, -371, 164, 1] + - ["CA", 1442037600000, -132, 8544, 1442030400000, 1442041200000, -371, 164, 1] + - ["CA", 1442041200000, 5, 8544, 1442034000000, 1442044800000, -371, 164, 1] + - ["CA", 1442044800000, 1, 8544, 1442037600000, 1442052000000, -371, 164, 2] + - ["CA", 1442052000000, 38, 8544, 1442041200000, 1442059200000, -371, 164, 2] + - ["CA", 1442059200000, 1036, 8544, 1442044800000, 1442062800000, -371, 164, 2] + - ["CA", 1442062800000, -367, 8544, 1442052000000, 1442066400000, -371, 164, 2] + - ["CA", 1442066400000, 307, 8544, 1442059200000, 1442070000000, -371, 164, 2] + - ["CA", 1442070000000, 185, 8544, 1442062800000, 1442073600000, -371, 164, 2] + - ["CA", 1442073600000, 86, 8544, 1442066400000, 1442077200000, -371, 164, 2] + - ["CA", 1442077200000, -282, 8544, 1442070000000, 1442080800000, -371, 164, 3] + - ["CA", 1442080800000, 481, 8544, 1442073600000, 1442084400000, -371, 164, 3] + - ["CA", 1442084400000, 44, 8544, 1442077200000, 1442088000000, -371, 164, 3] + - ["CA", 1442088000000, 35, 8544, 1442080800000, 1442091600000, -371, 164, 3] + - ["CA", 1442091600000, 2858, 8544, 1442084400000, 1442095200000, -371, 164, 3] + - ["CA", 1442095200000, -361, 8544, 1442088000000, 1442098800000, -371, 164, 3] + - ["CA", 1442098800000, 164, 8544, 1442091600000, null, -371, 164, 3] + - ["CH", 1442037600000, 59, 753, null, 1442041200000, 59, 67, 1] + - ["CH", 1442041200000, 198, 753, null, 1442044800000, 59, 67, 1] + - ["CH", 1442044800000, -54, 753, 1442037600000, 1442048400000, 59, 67, 1] + - ["CH", 1442048400000, 24, 753, 1442041200000, 1442052000000, 59, 67, 1] + - ["CH", 1442052000000, 47, 753, 1442044800000, 1442055600000, 59, 67, 2] + - ["CH", 1442055600000, 0, 753, 1442048400000, 1442062800000, 59, 67, 2] + - ["CH", 1442062800000, 22, 753, 1442052000000, 1442070000000, 59, 67, 2] + - ["CH", 1442070000000, 11, 753, 1442055600000, 1442073600000, 59, 67, 2] + - ["CH", 1442073600000, 360, 753, 1442062800000, 1442077200000, 59, 67, 3] + - ["CH", 1442077200000, 6, 753, 1442070000000, 1442084400000, 59, 67, 3] + - ["CH", 1442084400000, 13, 753, 1442073600000, 1442091600000, 59, 67, 3] + - ["CH", 1442091600000, 67, 753, 1442077200000, null, 59, 67, 3] + - ["CL", 1442016000000, 161, 533, null, 1442019600000, 161, 9, 1] + - ["CL", 1442019600000, -370, 533, null, 1442023200000, 161, 9, 1] + - ["CL", 1442023200000, 15, 533, 1442016000000, 1442030400000, 161, 9, 1] + - ["CL", 1442030400000, 40, 533, 1442019600000, 1442034000000, 161, 9, 1] + - ["CL", 1442034000000, -1, 533, 1442023200000, 1442037600000, 161, 9, 1] + - ["CL", 1442037600000, 2, 533, 1442030400000, 1442041200000, 161, 9, 1] + - ["CL", 1442041200000, -1, 533, 1442034000000, 1442052000000, 161, 9, 1] + - ["CL", 1442052000000, 390, 533, 1442037600000, 1442059200000, 161, 9, 2] + - ["CL", 1442059200000, -12, 533, 1442041200000, 1442062800000, 161, 9, 2] + - ["CL", 1442062800000, 17, 533, 1442052000000, 1442066400000, 161, 9, 2] + - ["CL", 1442066400000, -41, 533, 1442059200000, 1442070000000, 161, 9, 2] + - ["CL", 1442070000000, 13, 533, 1442062800000, 1442073600000, 161, 9, 2] + - ["CL", 1442073600000, 153, 533, 1442066400000, 1442077200000, 161, 9, 2] + - ["CL", 1442077200000, -15, 533, 1442070000000, 1442080800000, 161, 9, 2] + - ["CL", 1442080800000, 17, 533, 1442073600000, 1442084400000, 161, 9, 3] + - ["CL", 1442084400000, 126, 533, 1442077200000, 1442088000000, 161, 9, 3] + - ["CL", 1442088000000, 286, 533, 1442080800000, 1442091600000, 161, 9, 3] + - ["CL", 1442091600000, 20, 533, 1442084400000, 1442095200000, 161, 9, 3] + - ["CL", 1442095200000, -276, 533, 1442088000000, 1442098800000, 161, 9, 3] + - ["CL", 1442098800000, 9, 533, 1442091600000, null, 161, 9, 3] + - ["CN", 1442023200000, -13, 583, null, 1442026800000, -13, -1, 1] + - ["CN", 1442026800000, 154, 583, null, 1442037600000, -13, -1, 1] + - ["CN", 1442037600000, 98, 583, 1442023200000, 1442048400000, -13, -1, 1] + - ["CN", 1442048400000, 293, 583, 1442026800000, 1442052000000, -13, -1, 1] + - ["CN", 1442052000000, 0, 583, 1442037600000, 1442055600000, -13, -1, 2] + - ["CN", 1442055600000, 69, 583, 1442048400000, 1442059200000, -13, -1, 2] + - ["CN", 1442059200000, 8, 583, 1442052000000, 1442066400000, -13, -1, 2] + - ["CN", 1442066400000, -15, 583, 1442055600000, 1442080800000, -13, -1, 3] + - ["CN", 1442080800000, -10, 583, 1442059200000, 1442084400000, -13, -1, 3] + - ["CN", 1442084400000, -1, 583, 1442066400000, null, -13, -1, 3] + - ["CO", 1442016000000, 16, 59611, null, 1442019600000, 16, 83, 1] + - ["CO", 1442019600000, 12, 59611, null, 1442023200000, 16, 83, 1] + - ["CO", 1442023200000, 9, 59611, 1442016000000, 1442030400000, 16, 83, 1] + - ["CO", 1442030400000, 441, 59611, 1442019600000, 1442059200000, 16, 83, 1] + - ["CO", 1442059200000, 473, 59611, 1442023200000, 1442066400000, 16, 83, 1] + - ["CO", 1442066400000, 288, 59611, 1442030400000, 1442070000000, 16, 83, 2] + - ["CO", 1442070000000, -45, 59611, 1442059200000, 1442073600000, 16, 83, 2] + - ["CO", 1442073600000, 39860, 59611, 1442066400000, 1442077200000, 16, 83, 2] + - ["CO", 1442077200000, 581, 59611, 1442070000000, 1442080800000, 16, 83, 2] + - ["CO", 1442080800000, 25, 59611, 1442073600000, 1442084400000, 16, 83, 2] + - ["CO", 1442084400000, 51, 59611, 1442077200000, 1442088000000, 16, 83, 3] + - ["CO", 1442088000000, 17150, 59611, 1442080800000, 1442091600000, 16, 83, 3] + - ["CO", 1442091600000, 377, 59611, 1442084400000, 1442095200000, 16, 83, 3] + - ["CO", 1442095200000, 290, 59611, 1442088000000, 1442098800000, 16, 83, 3] + - ["CO", 1442098800000, 83, 59611, 1442091600000, null, 16, 83, 3] + - ["CR", 1442019600000, 62, 3241, null, 1442023200000, 62, 72, 1] + - ["CR", 1442023200000, 62, 3241, null, 1442026800000, 62, 72, 1] + - ["CR", 1442026800000, 140, 3241, 1442019600000, 1442030400000, 62, 72, 1] + - ["CR", 1442030400000, 2497, 3241, 1442023200000, 1442041200000, 62, 72, 2] + - ["CR", 1442041200000, 51, 3241, 1442026800000, 1442044800000, 62, 72, 2] + - ["CR", 1442044800000, 194, 3241, 1442030400000, 1442048400000, 62, 72, 2] + - ["CR", 1442048400000, 163, 3241, 1442041200000, 1442088000000, 62, 72, 3] + - ["CR", 1442088000000, 72, 3241, 1442044800000, null, 62, 72, 3] + - ["CZ", 1442026800000, -19, 3479, null, 1442034000000, -19, 2, 1] + - ["CZ", 1442034000000, 78, 3479, null, 1442037600000, -19, 2, 1] + - ["CZ", 1442037600000, 18, 3479, 1442026800000, 1442055600000, -19, 2, 1] + - ["CZ", 1442055600000, 1073, 3479, 1442034000000, 1442059200000, -19, 2, 1] + - ["CZ", 1442059200000, 21, 3479, 1442037600000, 1442062800000, -19, 2, 2] + - ["CZ", 1442062800000, 0, 3479, 1442055600000, 1442070000000, -19, 2, 2] + - ["CZ", 1442070000000, 168, 3479, 1442059200000, 1442073600000, -19, 2, 2] + - ["CZ", 1442073600000, 2051, 3479, 1442062800000, 1442077200000, -19, 2, 2] + - ["CZ", 1442077200000, 115, 3479, 1442070000000, 1442080800000, -19, 2, 3] + - ["CZ", 1442080800000, -28, 3479, 1442073600000, 1442098800000, -19, 2, 3] + - ["CZ", 1442098800000, 2, 3479, 1442077200000, null, -19, 2, 3] + - ["DE", 1442016000000, 167, 25583, null, 1442019600000, 167, 329, 1] + - ["DE", 1442019600000, 0, 25583, null, 1442023200000, 167, 329, 1] + - ["DE", 1442023200000, 64, 25583, 1442016000000, 1442030400000, 167, 329, 1] + - ["DE", 1442030400000, 373, 25583, 1442019600000, 1442034000000, 167, 329, 1] + - ["DE", 1442034000000, 358, 25583, 1442023200000, 1442037600000, 167, 329, 1] + - ["DE", 1442037600000, 544, 25583, 1442030400000, 1442041200000, 167, 329, 1] + - ["DE", 1442041200000, 197, 25583, 1442034000000, 1442044800000, 167, 329, 1] + - ["DE", 1442044800000, 979, 25583, 1442037600000, 1442048400000, 167, 329, 1] + - ["DE", 1442048400000, 811, 25583, 1442041200000, 1442052000000, 167, 329, 2] + - ["DE", 1442052000000, 1600, 25583, 1442044800000, 1442055600000, 167, 329, 2] + - ["DE", 1442055600000, 1523, 25583, 1442048400000, 1442059200000, 167, 329, 2] + - ["DE", 1442059200000, 289, 25583, 1442052000000, 1442062800000, 167, 329, 2] + - ["DE", 1442062800000, 283, 25583, 1442055600000, 1442066400000, 167, 329, 2] + - ["DE", 1442066400000, 1577, 25583, 1442059200000, 1442070000000, 167, 329, 2] + - ["DE", 1442070000000, 1666, 25583, 1442062800000, 1442073600000, 167, 329, 2] + - ["DE", 1442073600000, 6075, 25583, 1442066400000, 1442077200000, 167, 329, 2] + - ["DE", 1442077200000, 2188, 25583, 1442070000000, 1442080800000, 167, 329, 3] + - ["DE", 1442080800000, 1133, 25583, 1442073600000, 1442084400000, 167, 329, 3] + - ["DE", 1442084400000, -125, 25583, 1442077200000, 1442088000000, 167, 329, 3] + - ["DE", 1442088000000, 190, 25583, 1442080800000, 1442091600000, 167, 329, 3] + - ["DE", 1442091600000, 4355, 25583, 1442084400000, 1442095200000, 167, 329, 3] + - ["DE", 1442095200000, 1007, 25583, 1442088000000, 1442098800000, 167, 329, 3] + - ["DE", 1442098800000, 329, 25583, 1442091600000, null, 167, 329, 3] + - ["DK", 1442037600000, 10, 594, null, 1442044800000, 10, 0, 1] + - ["DK", 1442044800000, 36, 594, null, 1442048400000, 10, 0, 1] + - ["DK", 1442048400000, -5, 594, 1442037600000, 1442055600000, 10, 0, 1] + - ["DK", 1442055600000, 42, 594, 1442044800000, 1442059200000, 10, 0, 1] + - ["DK", 1442059200000, 0, 594, 1442048400000, 1442062800000, 10, 0, 2] + - ["DK", 1442062800000, 1, 594, 1442055600000, 1442066400000, 10, 0, 2] + - ["DK", 1442066400000, 416, 594, 1442059200000, 1442077200000, 10, 0, 2] + - ["DK", 1442077200000, -9, 594, 1442062800000, 1442080800000, 10, 0, 2] + - ["DK", 1442080800000, 61, 594, 1442066400000, 1442084400000, 10, 0, 3] + - ["DK", 1442084400000, -97, 594, 1442077200000, 1442091600000, 10, 0, 3] + - ["DK", 1442091600000, 139, 594, 1442080800000, 1442095200000, 10, 0, 3] + - ["DK", 1442095200000, 0, 594, 1442084400000, null, 10, 0, 3] + - ["DO", 1442023200000, 8, 264, null, 1442066400000, 8, 13, 1] + - ["DO", 1442066400000, 35, 264, null, 1442073600000, 8, 13, 1] + - ["DO", 1442073600000, 200, 264, 1442023200000, 1442084400000, 8, 13, 2] + - ["DO", 1442084400000, 8, 264, 1442066400000, 1442095200000, 8, 13, 2] + - ["DO", 1442095200000, 13, 264, 1442073600000, null, 8, 13, 3] + - ["DZ", 1442077200000, -1, -1, null, null, -1, -1, 1] + - ["EC", 1442019600000, 29, 232, null, 1442023200000, 29, 10, 1] + - ["EC", 1442023200000, -9, 232, null, 1442030400000, 29, 10, 1] + - ["EC", 1442030400000, 0, 232, 1442019600000, 1442077200000, 29, 10, 2] + - ["EC", 1442077200000, -366, 232, 1442023200000, 1442084400000, 29, 10, 2] + - ["EC", 1442084400000, 568, 232, 1442030400000, 1442095200000, 29, 10, 3] + - ["EC", 1442095200000, 10, 232, 1442077200000, null, 29, 10, 3] + - ["EE", 1442041200000, 37, 18, null, 1442044800000, 37, -19, 1] + - ["EE", 1442044800000, -19, 18, null, null, 37, -19, 2] + - ["EG", 1442026800000, 16, 170, null, 1442055600000, 16, 27, 1] + - ["EG", 1442055600000, 14, 170, null, 1442062800000, 16, 27, 1] + - ["EG", 1442062800000, 112, 170, 1442026800000, 1442073600000, 16, 27, 2] + - ["EG", 1442073600000, 1, 170, 1442055600000, 1442091600000, 16, 27, 2] + - ["EG", 1442091600000, 27, 170, 1442062800000, null, 16, 27, 3] + - ["ES", 1442019600000, 103, 7449, null, 1442023200000, 103, 458, 1] + - ["ES", 1442023200000, -5, 7449, null, 1442034000000, 103, 458, 1] + - ["ES", 1442034000000, -52, 7449, 1442019600000, 1442037600000, 103, 458, 1] + - ["ES", 1442037600000, 3, 7449, 1442023200000, 1442041200000, 103, 458, 1] + - ["ES", 1442041200000, 118, 7449, 1442034000000, 1442044800000, 103, 458, 1] + - ["ES", 1442044800000, -169, 7449, 1442037600000, 1442048400000, 103, 458, 1] + - ["ES", 1442048400000, 158, 7449, 1442041200000, 1442052000000, 103, 458, 1] + - ["ES", 1442052000000, -4, 7449, 1442044800000, 1442055600000, 103, 458, 2] + - ["ES", 1442055600000, 495, 7449, 1442048400000, 1442059200000, 103, 458, 2] + - ["ES", 1442059200000, 1086, 7449, 1442052000000, 1442062800000, 103, 458, 2] + - ["ES", 1442062800000, -71, 7449, 1442055600000, 1442066400000, 103, 458, 2] + - ["ES", 1442066400000, 461, 7449, 1442059200000, 1442070000000, 103, 458, 2] + - ["ES", 1442070000000, 61, 7449, 1442062800000, 1442073600000, 103, 458, 2] + - ["ES", 1442073600000, 154, 7449, 1442066400000, 1442077200000, 103, 458, 2] + - ["ES", 1442077200000, 1240, 7449, 1442070000000, 1442084400000, 103, 458, 3] + - ["ES", 1442084400000, 337, 7449, 1442073600000, 1442088000000, 103, 458, 3] + - ["ES", 1442088000000, -130, 7449, 1442077200000, 1442091600000, 103, 458, 3] + - ["ES", 1442091600000, 700, 7449, 1442084400000, 1442095200000, 103, 458, 3] + - ["ES", 1442095200000, 2506, 7449, 1442088000000, 1442098800000, 103, 458, 3] + - ["ES", 1442098800000, 458, 7449, 1442091600000, null, 103, 458, 3] + - ["FI", 1442030400000, 1491, 3579, null, 1442037600000, 1491, 69, 1] + - ["FI", 1442037600000, 14, 3579, null, 1442048400000, 1491, 69, 1] + - ["FI", 1442048400000, 12, 3579, 1442030400000, 1442052000000, 1491, 69, 1] + - ["FI", 1442052000000, 186, 3579, 1442037600000, 1442059200000, 1491, 69, 1] + - ["FI", 1442059200000, 407, 3579, 1442048400000, 1442062800000, 1491, 69, 2] + - ["FI", 1442062800000, 19, 3579, 1442052000000, 1442066400000, 1491, 69, 2] + - ["FI", 1442066400000, 183, 3579, 1442059200000, 1442073600000, 1491, 69, 2] + - ["FI", 1442073600000, -1, 3579, 1442062800000, 1442077200000, 1491, 69, 2] + - ["FI", 1442077200000, 200, 3579, 1442066400000, 1442080800000, 1491, 69, 3] + - ["FI", 1442080800000, 104, 3579, 1442073600000, 1442084400000, 1491, 69, 3] + - ["FI", 1442084400000, 895, 3579, 1442077200000, 1442095200000, 1491, 69, 3] + - ["FI", 1442095200000, 69, 3579, 1442080800000, null, 1491, 69, 3] + - ["FR", 1442016000000, -1, 37281, null, 1442019600000, -1, 136, 1] + - ["FR", 1442019600000, 585, 37281, null, 1442023200000, -1, 136, 1] + - ["FR", 1442023200000, 628, 37281, 1442016000000, 1442026800000, -1, 136, 1] + - ["FR", 1442026800000, 86, 37281, 1442019600000, 1442034000000, -1, 136, 1] + - ["FR", 1442034000000, 476, 37281, 1442023200000, 1442037600000, -1, 136, 1] + - ["FR", 1442037600000, 4174, 37281, 1442026800000, 1442041200000, -1, 136, 1] + - ["FR", 1442041200000, 604, 37281, 1442034000000, 1442044800000, -1, 136, 1] + - ["FR", 1442044800000, 172, 37281, 1442037600000, 1442048400000, -1, 136, 1] + - ["FR", 1442048400000, 3027, 37281, 1442041200000, 1442052000000, -1, 136, 2] + - ["FR", 1442052000000, 637, 37281, 1442044800000, 1442055600000, -1, 136, 2] + - ["FR", 1442055600000, 463, 37281, 1442048400000, 1442059200000, -1, 136, 2] + - ["FR", 1442059200000, 4650, 37281, 1442052000000, 1442062800000, -1, 136, 2] + - ["FR", 1442062800000, 5676, 37281, 1442055600000, 1442066400000, -1, 136, 2] + - ["FR", 1442066400000, 2516, 37281, 1442059200000, 1442070000000, -1, 136, 2] + - ["FR", 1442070000000, 474, 37281, 1442062800000, 1442073600000, -1, 136, 2] + - ["FR", 1442073600000, 3522, 37281, 1442066400000, 1442077200000, -1, 136, 2] + - ["FR", 1442077200000, -444, 37281, 1442070000000, 1442080800000, -1, 136, 3] + - ["FR", 1442080800000, 557, 37281, 1442073600000, 1442084400000, -1, 136, 3] + - ["FR", 1442084400000, 6643, 37281, 1442077200000, 1442088000000, -1, 136, 3] + - ["FR", 1442088000000, 1872, 37281, 1442080800000, 1442091600000, -1, 136, 3] + - ["FR", 1442091600000, 741, 37281, 1442084400000, 1442095200000, -1, 136, 3] + - ["FR", 1442095200000, 87, 37281, 1442088000000, 1442098800000, -1, 136, 3] + - ["FR", 1442098800000, 136, 37281, 1442091600000, null, -1, 136, 3] + - ["GB", 1442016000000, -44, 35857, null, 1442019600000, -44, 49, 1] + - ["GB", 1442019600000, 54, 35857, null, 1442023200000, -44, 49, 1] + - ["GB", 1442023200000, 1816, 35857, 1442016000000, 1442026800000, -44, 49, 1] + - ["GB", 1442026800000, 339, 35857, 1442019600000, 1442030400000, -44, 49, 1] + - ["GB", 1442030400000, 2524, 35857, 1442023200000, 1442034000000, -44, 49, 1] + - ["GB", 1442034000000, -12, 35857, 1442026800000, 1442037600000, -44, 49, 1] + - ["GB", 1442037600000, 544, 35857, 1442030400000, 1442041200000, -44, 49, 1] + - ["GB", 1442041200000, 42, 35857, 1442034000000, 1442044800000, -44, 49, 1] + - ["GB", 1442044800000, 32, 35857, 1442037600000, 1442048400000, -44, 49, 2] + - ["GB", 1442048400000, 740, 35857, 1442041200000, 1442052000000, -44, 49, 2] + - ["GB", 1442052000000, 168, 35857, 1442044800000, 1442055600000, -44, 49, 2] + - ["GB", 1442055600000, 453, 35857, 1442048400000, 1442059200000, -44, 49, 2] + - ["GB", 1442059200000, 16111, 35857, 1442052000000, 1442062800000, -44, 49, 2] + - ["GB", 1442062800000, 5743, 35857, 1442055600000, 1442066400000, -44, 49, 2] + - ["GB", 1442066400000, 671, 35857, 1442059200000, 1442070000000, -44, 49, 2] + - ["GB", 1442070000000, 374, 35857, 1442062800000, 1442073600000, -44, 49, 2] + - ["GB", 1442073600000, 648, 35857, 1442066400000, 1442077200000, -44, 49, 3] + - ["GB", 1442077200000, 1135, 35857, 1442070000000, 1442080800000, -44, 49, 3] + - ["GB", 1442080800000, 1444, 35857, 1442073600000, 1442084400000, -44, 49, 3] + - ["GB", 1442084400000, 384, 35857, 1442077200000, 1442088000000, -44, 49, 3] + - ["GB", 1442088000000, 1593, 35857, 1442080800000, 1442091600000, -44, 49, 3] + - ["GB", 1442091600000, 811, 35857, 1442084400000, 1442095200000, -44, 49, 3] + - ["GB", 1442095200000, 238, 35857, 1442088000000, 1442098800000, -44, 49, 3] + - ["GB", 1442098800000, 49, 35857, 1442091600000, null, -44, 49, 3] + - ["GE", 1442044800000, -21, -140, null, 1442052000000, -21, -27, 1] + - ["GE", 1442052000000, -108, -140, null, 1442062800000, -21, -27, 1] + - ["GE", 1442062800000, 16, -140, 1442044800000, 1442080800000, -21, -27, 2] + - ["GE", 1442080800000, -27, -140, 1442052000000, null, -21, -27, 3] + - ["GH", 1442088000000, 0, 0, null, null, 0, 0, 1] + - ["GR", 1442019600000, 82, 149, null, 1442034000000, 82, 123, 1] + - ["GR", 1442034000000, 0, 149, null, 1442041200000, 82, 123, 1] + - ["GR", 1442041200000, 7, 149, 1442019600000, 1442048400000, 82, 123, 1] + - ["GR", 1442048400000, -26, 149, 1442034000000, 1442062800000, 82, 123, 1] + - ["GR", 1442062800000, 8, 149, 1442041200000, 1442070000000, 82, 123, 2] + - ["GR", 1442070000000, 2, 149, 1442048400000, 1442073600000, 82, 123, 2] + - ["GR", 1442073600000, -314, 149, 1442062800000, 1442080800000, 82, 123, 2] + - ["GR", 1442080800000, 88, 149, 1442070000000, 1442084400000, 82, 123, 3] + - ["GR", 1442084400000, 179, 149, 1442073600000, 1442091600000, 82, 123, 3] + - ["GR", 1442091600000, 123, 149, 1442080800000, null, 82, 123, 3] + - ["GT", 1442023200000, -167, 7, null, 1442026800000, -167, 1, 1] + - ["GT", 1442026800000, 173, 7, null, 1442098800000, -167, 1, 2] + - ["GT", 1442098800000, 1, 7, 1442023200000, null, -167, 1, 3] + - ["HK", 1442019600000, -113, 10743, null, 1442023200000, -113, -1, 1] + - ["HK", 1442023200000, 2414, 10743, null, 1442026800000, -113, -1, 1] + - ["HK", 1442026800000, -211, 10743, 1442019600000, 1442030400000, -113, -1, 1] + - ["HK", 1442030400000, 157, 10743, 1442023200000, 1442034000000, -113, -1, 1] + - ["HK", 1442034000000, 1137, 10743, 1442026800000, 1442037600000, -113, -1, 1] + - ["HK", 1442037600000, 636, 10743, 1442030400000, 1442041200000, -113, -1, 1] + - ["HK", 1442041200000, -15, 10743, 1442034000000, 1442044800000, -113, -1, 1] + - ["HK", 1442044800000, 21, 10743, 1442037600000, 1442048400000, -113, -1, 2] + - ["HK", 1442048400000, 1, 10743, 1442041200000, 1442052000000, -113, -1, 2] + - ["HK", 1442052000000, 15, 10743, 1442044800000, 1442055600000, -113, -1, 2] + - ["HK", 1442055600000, 804, 10743, 1442048400000, 1442059200000, -113, -1, 2] + - ["HK", 1442059200000, 2, 10743, 1442052000000, 1442062800000, -113, -1, 2] + - ["HK", 1442062800000, 1, 10743, 1442055600000, 1442066400000, -113, -1, 2] + - ["HK", 1442066400000, 39, 10743, 1442059200000, 1442070000000, -113, -1, 3] + - ["HK", 1442070000000, 314, 10743, 1442062800000, 1442073600000, -113, -1, 3] + - ["HK", 1442073600000, 5545, 10743, 1442066400000, 1442080800000, -113, -1, 3] + - ["HK", 1442080800000, 0, 10743, 1442070000000, 1442091600000, -113, -1, 3] + - ["HK", 1442091600000, -3, 10743, 1442073600000, 1442095200000, -113, -1, 3] + - ["HK", 1442095200000, -1, 10743, 1442080800000, null, -113, -1, 3] + - ["HN", 1442026800000, -1, -1, null, null, -1, -1, 1] + - ["HR", 1442070000000, 32, 382, null, 1442073600000, 32, 82, 1] + - ["HR", 1442073600000, 0, 382, null, 1442077200000, 32, 82, 1] + - ["HR", 1442077200000, 58, 382, 1442070000000, 1442080800000, 32, 82, 2] + - ["HR", 1442080800000, 220, 382, 1442073600000, 1442084400000, 32, 82, 2] + - ["HR", 1442084400000, -10, 382, 1442077200000, 1442088000000, 32, 82, 3] + - ["HR", 1442088000000, 82, 382, 1442080800000, null, 32, 82, 3] + - ["HU", 1442019600000, 46, 2116, null, 1442037600000, 46, 110, 1] + - ["HU", 1442037600000, 197, 2116, null, 1442041200000, 46, 110, 1] + - ["HU", 1442041200000, 91, 2116, 1442019600000, 1442044800000, 46, 110, 1] + - ["HU", 1442044800000, 547, 2116, 1442037600000, 1442048400000, 46, 110, 1] + - ["HU", 1442048400000, 499, 2116, 1442041200000, 1442055600000, 46, 110, 1] + - ["HU", 1442055600000, -2, 2116, 1442044800000, 1442062800000, 46, 110, 2] + - ["HU", 1442062800000, 50, 2116, 1442048400000, 1442080800000, 46, 110, 2] + - ["HU", 1442080800000, 242, 2116, 1442055600000, 1442084400000, 46, 110, 2] + - ["HU", 1442084400000, 141, 2116, 1442062800000, 1442088000000, 46, 110, 2] + - ["HU", 1442088000000, -71, 2116, 1442080800000, 1442091600000, 46, 110, 3] + - ["HU", 1442091600000, -5, 2116, 1442084400000, 1442095200000, 46, 110, 3] + - ["HU", 1442095200000, 271, 2116, 1442088000000, 1442098800000, 46, 110, 3] + - ["HU", 1442098800000, 110, 2116, 1442091600000, null, 46, 110, 3] + - ["ID", 1442023200000, 106, -255, null, 1442026800000, 106, 13, 1] + - ["ID", 1442026800000, -416, -255, null, 1442030400000, 106, 13, 1] + - ["ID", 1442030400000, 279, -255, 1442023200000, 1442034000000, 106, 13, 1] + - ["ID", 1442034000000, 19, -255, 1442026800000, 1442037600000, 106, 13, 1] + - ["ID", 1442037600000, 14, -255, 1442030400000, 1442041200000, 106, 13, 1] + - ["ID", 1442041200000, 2, -255, 1442034000000, 1442044800000, 106, 13, 2] + - ["ID", 1442044800000, -388, -255, 1442037600000, 1442055600000, 106, 13, 2] + - ["ID", 1442055600000, 16, -255, 1442041200000, 1442059200000, 106, 13, 2] + - ["ID", 1442059200000, 17, -255, 1442044800000, 1442070000000, 106, 13, 2] + - ["ID", 1442070000000, 42, -255, 1442055600000, 1442091600000, 106, 13, 3] + - ["ID", 1442091600000, 21, -255, 1442059200000, 1442095200000, 106, 13, 3] + - ["ID", 1442095200000, 20, -255, 1442070000000, 1442098800000, 106, 13, 3] + - ["ID", 1442098800000, 13, -255, 1442091600000, null, 106, 13, 3] + - ["IE", 1442026800000, 1, 2142, null, 1442030400000, 1, -71, 1] + - ["IE", 1442030400000, 1, 2142, null, 1442048400000, 1, -71, 1] + - ["IE", 1442048400000, 27, 2142, 1442026800000, 1442066400000, 1, -71, 1] + - ["IE", 1442066400000, 1062, 2142, 1442030400000, 1442070000000, 1, -71, 2] + - ["IE", 1442070000000, -100, 2142, 1442048400000, 1442077200000, 1, -71, 2] + - ["IE", 1442077200000, 403, 2142, 1442066400000, 1442084400000, 1, -71, 2] + - ["IE", 1442084400000, 819, 2142, 1442070000000, 1442091600000, 1, -71, 3] + - ["IE", 1442091600000, -71, 2142, 1442077200000, null, 1, -71, 3] + - ["IL", 1442041200000, 35, 6617, null, 1442044800000, 35, 3, 1] + - ["IL", 1442044800000, 218, 6617, null, 1442048400000, 35, 3, 1] + - ["IL", 1442048400000, 25, 6617, 1442041200000, 1442052000000, 35, 3, 1] + - ["IL", 1442052000000, 2745, 6617, 1442044800000, 1442055600000, 35, 3, 1] + - ["IL", 1442055600000, 4, 6617, 1442048400000, 1442059200000, 35, 3, 1] + - ["IL", 1442059200000, 1205, 6617, 1442052000000, 1442062800000, 35, 3, 1] + - ["IL", 1442062800000, 180, 6617, 1442055600000, 1442066400000, 35, 3, 2] + - ["IL", 1442066400000, 3, 6617, 1442059200000, 1442070000000, 35, 3, 2] + - ["IL", 1442070000000, 49, 6617, 1442062800000, 1442073600000, 35, 3, 2] + - ["IL", 1442073600000, 31, 6617, 1442066400000, 1442077200000, 35, 3, 2] + - ["IL", 1442077200000, 187, 6617, 1442070000000, 1442080800000, 35, 3, 2] + - ["IL", 1442080800000, 88, 6617, 1442073600000, 1442084400000, 35, 3, 3] + - ["IL", 1442084400000, 1137, 6617, 1442077200000, 1442091600000, 35, 3, 3] + - ["IL", 1442091600000, 707, 6617, 1442080800000, 1442095200000, 35, 3, 3] + - ["IL", 1442095200000, 0, 6617, 1442084400000, 1442098800000, 35, 3, 3] + - ["IL", 1442098800000, 3, 6617, 1442091600000, null, 35, 3, 3] + - ["IN", 1442016000000, 1, 29166, null, 1442019600000, 1, 4, 1] + - ["IN", 1442019600000, 38, 29166, null, 1442023200000, 1, 4, 1] + - ["IN", 1442023200000, -142, 29166, 1442016000000, 1442026800000, 1, 4, 1] + - ["IN", 1442026800000, 974, 29166, 1442019600000, 1442030400000, 1, 4, 1] + - ["IN", 1442030400000, 1448, 29166, 1442023200000, 1442034000000, 1, 4, 1] + - ["IN", 1442034000000, 1350, 29166, 1442026800000, 1442037600000, 1, 4, 1] + - ["IN", 1442037600000, 135, 29166, 1442030400000, 1442041200000, 1, 4, 1] + - ["IN", 1442041200000, 80, 29166, 1442034000000, 1442044800000, 1, 4, 1] + - ["IN", 1442044800000, 2677, 29166, 1442037600000, 1442048400000, 1, 4, 2] + - ["IN", 1442048400000, 262, 29166, 1442041200000, 1442052000000, 1, 4, 2] + - ["IN", 1442052000000, 534, 29166, 1442044800000, 1442055600000, 1, 4, 2] + - ["IN", 1442055600000, 166, 29166, 1442048400000, 1442059200000, 1, 4, 2] + - ["IN", 1442059200000, 708, 29166, 1442052000000, 1442062800000, 1, 4, 2] + - ["IN", 1442062800000, 1547, 29166, 1442055600000, 1442066400000, 1, 4, 2] + - ["IN", 1442066400000, 116, 29166, 1442059200000, 1442070000000, 1, 4, 2] + - ["IN", 1442070000000, 12091, 29166, 1442062800000, 1442073600000, 1, 4, 3] + - ["IN", 1442073600000, 1170, 29166, 1442066400000, 1442077200000, 1, 4, 3] + - ["IN", 1442077200000, 5699, 29166, 1442070000000, 1442080800000, 1, 4, 3] + - ["IN", 1442080800000, 0, 29166, 1442073600000, 1442084400000, 1, 4, 3] + - ["IN", 1442084400000, 187, 29166, 1442077200000, 1442088000000, 1, 4, 3] + - ["IN", 1442088000000, 121, 29166, 1442080800000, 1442095200000, 1, 4, 3] + - ["IN", 1442095200000, 4, 29166, 1442084400000, null, 1, 4, 3] + - ["IQ", 1442041200000, -1, 3, null, 1442044800000, -1, -2, 1] + - ["IQ", 1442044800000, 6, 3, null, 1442052000000, -1, -2, 1] + - ["IQ", 1442052000000, 0, 3, 1442041200000, 1442095200000, -1, -2, 2] + - ["IQ", 1442095200000, -2, 3, 1442044800000, null, -1, -2, 3] + - ["IR", 1442026800000, 0, 2698, null, 1442030400000, 0, 0, 1] + - ["IR", 1442030400000, 375, 2698, null, 1442034000000, 0, 0, 1] + - ["IR", 1442034000000, -8, 2698, 1442026800000, 1442041200000, 0, 0, 1] + - ["IR", 1442041200000, -79, 2698, 1442030400000, 1442044800000, 0, 0, 1] + - ["IR", 1442044800000, 306, 2698, 1442034000000, 1442052000000, 0, 0, 1] + - ["IR", 1442052000000, 155, 2698, 1442041200000, 1442055600000, 0, 0, 2] + - ["IR", 1442055600000, -124, 2698, 1442044800000, 1442059200000, 0, 0, 2] + - ["IR", 1442059200000, 1455, 2698, 1442052000000, 1442073600000, 0, 0, 2] + - ["IR", 1442073600000, -193, 2698, 1442055600000, 1442077200000, 0, 0, 2] + - ["IR", 1442077200000, -34, 2698, 1442059200000, 1442080800000, 0, 0, 3] + - ["IR", 1442080800000, 131, 2698, 1442073600000, 1442088000000, 0, 0, 3] + - ["IR", 1442088000000, 714, 2698, 1442077200000, 1442091600000, 0, 0, 3] + - ["IR", 1442091600000, 0, 2698, 1442080800000, null, 0, 0, 3] + - ["IT", 1442016000000, 0, 39091, null, 1442019600000, 0, 565, 1] + - ["IT", 1442019600000, 183, 39091, null, 1442023200000, 0, 565, 1] + - ["IT", 1442023200000, 111, 39091, 1442016000000, 1442026800000, 0, 565, 1] + - ["IT", 1442026800000, 222, 39091, 1442019600000, 1442030400000, 0, 565, 1] + - ["IT", 1442030400000, -17, 39091, 1442023200000, 1442034000000, 0, 565, 1] + - ["IT", 1442034000000, 1006, 39091, 1442026800000, 1442037600000, 0, 565, 1] + - ["IT", 1442037600000, -9, 39091, 1442030400000, 1442041200000, 0, 565, 1] + - ["IT", 1442041200000, 20, 39091, 1442034000000, 1442044800000, 0, 565, 1] + - ["IT", 1442044800000, 1483, 39091, 1442037600000, 1442048400000, 0, 565, 2] + - ["IT", 1442048400000, 676, 39091, 1442041200000, 1442052000000, 0, 565, 2] + - ["IT", 1442052000000, 1880, 39091, 1442044800000, 1442055600000, 0, 565, 2] + - ["IT", 1442055600000, 6240, 39091, 1442048400000, 1442059200000, 0, 565, 2] + - ["IT", 1442059200000, 542, 39091, 1442052000000, 1442062800000, 0, 565, 2] + - ["IT", 1442062800000, 1938, 39091, 1442055600000, 1442066400000, 0, 565, 2] + - ["IT", 1442066400000, 4155, 39091, 1442059200000, 1442070000000, 0, 565, 2] + - ["IT", 1442070000000, 81, 39091, 1442062800000, 1442073600000, 0, 565, 2] + - ["IT", 1442073600000, 2586, 39091, 1442066400000, 1442077200000, 0, 565, 3] + - ["IT", 1442077200000, 2188, 39091, 1442070000000, 1442080800000, 0, 565, 3] + - ["IT", 1442080800000, 5544, 39091, 1442073600000, 1442084400000, 0, 565, 3] + - ["IT", 1442084400000, 2660, 39091, 1442077200000, 1442088000000, 0, 565, 3] + - ["IT", 1442088000000, 3746, 39091, 1442080800000, 1442091600000, 0, 565, 3] + - ["IT", 1442091600000, 351, 39091, 1442084400000, 1442095200000, 0, 565, 3] + - ["IT", 1442095200000, 2940, 39091, 1442088000000, 1442098800000, 0, 565, 3] + - ["IT", 1442098800000, 565, 39091, 1442091600000, null, 0, 565, 3] + - ["JM", 1442070000000, 30, 30, null, null, 30, 30, 1] + - ["JO", 1442055600000, -2, 2, null, 1442059200000, -2, 4, 1] + - ["JO", 1442059200000, 0, 2, null, 1442080800000, -2, 4, 2] + - ["JO", 1442080800000, 4, 2, 1442055600000, null, -2, 4, 3] + - ["JP", 1442016000000, -113, 20378, null, 1442019600000, -113, -6, 1] + - ["JP", 1442019600000, 2002, 20378, null, 1442023200000, -113, -6, 1] + - ["JP", 1442023200000, 1959, 20378, 1442016000000, 1442026800000, -113, -6, 1] + - ["JP", 1442026800000, 1035, 20378, 1442019600000, 1442030400000, -113, -6, 1] + - ["JP", 1442030400000, 805, 20378, 1442023200000, 1442034000000, -113, -6, 1] + - ["JP", 1442034000000, 910, 20378, 1442026800000, 1442037600000, -113, -6, 1] + - ["JP", 1442037600000, 2181, 20378, 1442030400000, 1442041200000, -113, -6, 1] + - ["JP", 1442041200000, 1373, 20378, 1442034000000, 1442044800000, -113, -6, 1] + - ["JP", 1442044800000, 1569, 20378, 1442037600000, 1442048400000, -113, -6, 2] + - ["JP", 1442048400000, 1981, 20378, 1442041200000, 1442052000000, -113, -6, 2] + - ["JP", 1442052000000, 2789, 20378, 1442044800000, 1442055600000, -113, -6, 2] + - ["JP", 1442055600000, 998, 20378, 1442048400000, 1442059200000, -113, -6, 2] + - ["JP", 1442059200000, -85, 20378, 1442052000000, 1442062800000, -113, -6, 2] + - ["JP", 1442062800000, 803, 20378, 1442055600000, 1442066400000, -113, -6, 2] + - ["JP", 1442066400000, 167, 20378, 1442059200000, 1442070000000, -113, -6, 2] + - ["JP", 1442070000000, 79, 20378, 1442062800000, 1442073600000, -113, -6, 2] + - ["JP", 1442073600000, 1162, 20378, 1442066400000, 1442077200000, -113, -6, 3] + - ["JP", 1442077200000, 51, 20378, 1442070000000, 1442080800000, -113, -6, 3] + - ["JP", 1442080800000, 420, 20378, 1442073600000, 1442084400000, -113, -6, 3] + - ["JP", 1442084400000, 13, 20378, 1442077200000, 1442088000000, -113, -6, 3] + - ["JP", 1442088000000, 57, 20378, 1442080800000, 1442091600000, -113, -6, 3] + - ["JP", 1442091600000, 228, 20378, 1442084400000, 1442095200000, -113, -6, 3] + - ["JP", 1442095200000, 0, 20378, 1442088000000, 1442098800000, -113, -6, 3] + - ["JP", 1442098800000, -6, 20378, 1442091600000, null, -113, -6, 3] + - ["KE", 1442044800000, -1, -1, null, null, -1, -1, 1] + - ["KG", 1442073600000, 6, 6, null, null, 6, 6, 1] + - ["KR", 1442016000000, 1024, 13597, null, 1442019600000, 1024, -36, 1] + - ["KR", 1442019600000, 445, 13597, null, 1442023200000, 1024, -36, 1] + - ["KR", 1442023200000, 319, 13597, 1442016000000, 1442026800000, 1024, -36, 1] + - ["KR", 1442026800000, -179, 13597, 1442019600000, 1442030400000, 1024, -36, 1] + - ["KR", 1442030400000, 1035, 13597, 1442023200000, 1442034000000, 1024, -36, 1] + - ["KR", 1442034000000, 434, 13597, 1442026800000, 1442037600000, 1024, -36, 1] + - ["KR", 1442037600000, 26, 13597, 1442030400000, 1442041200000, 1024, -36, 1] + - ["KR", 1442041200000, 20, 13597, 1442034000000, 1442044800000, 1024, -36, 1] + - ["KR", 1442044800000, 829, 13597, 1442037600000, 1442048400000, 1024, -36, 2] + - ["KR", 1442048400000, -374, 13597, 1442041200000, 1442052000000, 1024, -36, 2] + - ["KR", 1442052000000, -3, 13597, 1442044800000, 1442055600000, 1024, -36, 2] + - ["KR", 1442055600000, 3640, 13597, 1442048400000, 1442059200000, 1024, -36, 2] + - ["KR", 1442059200000, 208, 13597, 1442052000000, 1442062800000, 1024, -36, 2] + - ["KR", 1442062800000, 1096, 13597, 1442055600000, 1442066400000, 1024, -36, 2] + - ["KR", 1442066400000, 3299, 13597, 1442059200000, 1442070000000, 1024, -36, 2] + - ["KR", 1442070000000, 222, 13597, 1442062800000, 1442077200000, 1024, -36, 3] + - ["KR", 1442077200000, -40, 13597, 1442066400000, 1442080800000, 1024, -36, 3] + - ["KR", 1442080800000, -33, 13597, 1442070000000, 1442084400000, 1024, -36, 3] + - ["KR", 1442084400000, 314, 13597, 1442077200000, 1442088000000, 1024, -36, 3] + - ["KR", 1442088000000, 524, 13597, 1442080800000, 1442095200000, 1024, -36, 3] + - ["KR", 1442095200000, 827, 13597, 1442084400000, 1442098800000, 1024, -36, 3] + - ["KR", 1442098800000, -36, 13597, 1442088000000, null, 1024, -36, 3] + - ["KW", 1442055600000, -2, 1778, null, 1442070000000, -2, -33, 1] + - ["KW", 1442070000000, 1815, 1778, null, 1442077200000, -2, -33, 1] + - ["KW", 1442077200000, -2, 1778, 1442055600000, 1442080800000, -2, -33, 2] + - ["KW", 1442080800000, -33, 1778, 1442070000000, null, -2, -33, 3] + - ["KZ", 1442034000000, 161, 1261, null, 1442044800000, 161, 91, 1] + - ["KZ", 1442044800000, 401, 1261, null, 1442048400000, 161, 91, 1] + - ["KZ", 1442048400000, 439, 1261, 1442034000000, 1442052000000, 161, 91, 1] + - ["KZ", 1442052000000, 412, 1261, 1442044800000, 1442055600000, 161, 91, 1] + - ["KZ", 1442055600000, 63, 1261, 1442048400000, 1442059200000, 161, 91, 2] + - ["KZ", 1442059200000, 33, 1261, 1442052000000, 1442062800000, 161, 91, 2] + - ["KZ", 1442062800000, 0, 1261, 1442055600000, 1442066400000, 161, 91, 2] + - ["KZ", 1442066400000, 0, 1261, 1442059200000, 1442077200000, 161, 91, 2] + - ["KZ", 1442077200000, -317, 1261, 1442062800000, 1442084400000, 161, 91, 3] + - ["KZ", 1442084400000, -22, 1261, 1442066400000, 1442095200000, 161, 91, 3] + - ["KZ", 1442095200000, 91, 1261, 1442077200000, null, 161, 91, 3] + - ["LB", 1442055600000, -67, -67, null, null, -67, -67, 1] + - ["LK", 1442026800000, 79, 131, null, 1442048400000, 79, -3, 1] + - ["LK", 1442048400000, 8, 131, null, 1442052000000, 79, -3, 1] + - ["LK", 1442052000000, 47, 131, 1442026800000, 1442084400000, 79, -3, 2] + - ["LK", 1442084400000, -3, 131, 1442048400000, null, 79, -3, 3] + - ["LT", 1442080800000, 12, -12, null, 1442098800000, 12, -24, 1] + - ["LT", 1442098800000, -24, -12, null, null, 12, -24, 2] + - ["LU", 1442059200000, 79, 606, null, 1442066400000, 79, 2, 1] + - ["LU", 1442066400000, 0, 606, null, 1442077200000, 79, 2, 1] + - ["LU", 1442077200000, 525, 606, 1442059200000, 1442095200000, 79, 2, 2] + - ["LU", 1442095200000, 2, 606, 1442066400000, null, 79, 2, 3] + - ["LV", 1442095200000, 0, 0, null, null, 0, 0, 1] + - ["MA", 1442019600000, -1, 229, null, 1442055600000, -1, 8, 1] + - ["MA", 1442055600000, 23, 229, null, 1442059200000, -1, 8, 1] + - ["MA", 1442059200000, -56, 229, 1442019600000, 1442062800000, -1, 8, 1] + - ["MA", 1442062800000, 0, 229, 1442055600000, 1442077200000, -1, 8, 2] + - ["MA", 1442077200000, 250, 229, 1442059200000, 1442080800000, -1, 8, 2] + - ["MA", 1442080800000, 5, 229, 1442062800000, 1442098800000, -1, 8, 3] + - ["MA", 1442098800000, 8, 229, 1442077200000, null, -1, 8, 3] + - ["MD", 1442077200000, 6916, 6916, null, null, 6916, 6916, 1] + - ["ME", 1442073600000, 0, 0, null, null, 0, 0, 1] + - ["MH", 1442052000000, 40, 40, null, null, 40, 40, 1] + - ["MK", 1442077200000, -72, -72, null, null, -72, -72, 1] + - ["MM", 1442070000000, 3, 28, null, 1442073600000, 3, 25, 1] + - ["MM", 1442073600000, 25, 28, null, null, 3, 25, 2] + - ["MO", 1442034000000, 30, 48, null, 1442070000000, 30, 18, 1] + - ["MO", 1442070000000, 18, 48, null, null, 30, 18, 2] + - ["MR", 1442080800000, 10, 10, null, null, 10, 10, 1] + - ["MT", 1442048400000, -1, -1, null, null, -1, -1, 1] + - ["MV", 1442073600000, -3, -3, null, null, -3, -3, 1] + - ["MX", 1442016000000, -67, 10472, null, 1442023200000, -67, 28, 1] + - ["MX", 1442023200000, 549, 10472, null, 1442026800000, -67, 28, 1] + - ["MX", 1442026800000, 3642, 10472, 1442016000000, 1442030400000, -67, 28, 1] + - ["MX", 1442030400000, 373, 10472, 1442023200000, 1442034000000, -67, 28, 1] + - ["MX", 1442034000000, 944, 10472, 1442026800000, 1442037600000, -67, 28, 1] + - ["MX", 1442037600000, 4, 10472, 1442030400000, 1442041200000, -67, 28, 1] + - ["MX", 1442041200000, -294, 10472, 1442034000000, 1442066400000, -67, 28, 2] + - ["MX", 1442066400000, -1, 10472, 1442037600000, 1442070000000, -67, 28, 2] + - ["MX", 1442070000000, -1, 10472, 1442041200000, 1442073600000, -67, 28, 2] + - ["MX", 1442073600000, -21, 10472, 1442066400000, 1442077200000, -67, 28, 2] + - ["MX", 1442077200000, 3874, 10472, 1442070000000, 1442080800000, -67, 28, 2] + - ["MX", 1442080800000, -376, 10472, 1442073600000, 1442084400000, -67, 28, 2] + - ["MX", 1442084400000, 981, 10472, 1442077200000, 1442088000000, -67, 28, 3] + - ["MX", 1442088000000, 494, 10472, 1442080800000, 1442091600000, -67, 28, 3] + - ["MX", 1442091600000, 799, 10472, 1442084400000, 1442095200000, -67, 28, 3] + - ["MX", 1442095200000, -456, 10472, 1442088000000, 1442098800000, -67, 28, 3] + - ["MX", 1442098800000, 28, 10472, 1442091600000, null, -67, 28, 3] + - ["MY", 1442019600000, -7, 3207, null, 1442030400000, -7, 739, 1] + - ["MY", 1442030400000, -3, 3207, null, 1442034000000, -7, 739, 1] + - ["MY", 1442034000000, 1028, 3207, 1442019600000, 1442041200000, -7, 739, 1] + - ["MY", 1442041200000, 935, 3207, 1442030400000, 1442044800000, -7, 739, 1] + - ["MY", 1442044800000, -127, 3207, 1442034000000, 1442048400000, -7, 739, 2] + - ["MY", 1442048400000, 649, 3207, 1442041200000, 1442055600000, -7, 739, 2] + - ["MY", 1442055600000, 1, 3207, 1442044800000, 1442059200000, -7, 739, 2] + - ["MY", 1442059200000, 0, 3207, 1442048400000, 1442066400000, -7, 739, 2] + - ["MY", 1442066400000, 1, 3207, 1442055600000, 1442073600000, -7, 739, 3] + - ["MY", 1442073600000, 1, 3207, 1442059200000, 1442077200000, -7, 739, 3] + - ["MY", 1442077200000, -10, 3207, 1442066400000, 1442098800000, -7, 739, 3] + - ["MY", 1442098800000, 739, 3207, 1442073600000, null, -7, 739, 3] + - ["NG", 1442052000000, 208, 214, null, 1442070000000, 208, 6, 1] + - ["NG", 1442070000000, 6, 214, null, null, 208, 6, 2] + - ["NL", 1442034000000, 0, 12162, null, 1442044800000, 0, 4, 1] + - ["NL", 1442044800000, 16, 12162, null, 1442048400000, 0, 4, 1] + - ["NL", 1442048400000, 1303, 12162, 1442034000000, 1442052000000, 0, 4, 1] + - ["NL", 1442052000000, 53, 12162, 1442044800000, 1442055600000, 0, 4, 1] + - ["NL", 1442055600000, 105, 12162, 1442048400000, 1442059200000, 0, 4, 1] + - ["NL", 1442059200000, 206, 12162, 1442052000000, 1442062800000, 0, 4, 1] + - ["NL", 1442062800000, -30, 12162, 1442055600000, 1442066400000, 0, 4, 2] + - ["NL", 1442066400000, 61, 12162, 1442059200000, 1442070000000, 0, 4, 2] + - ["NL", 1442070000000, -84, 12162, 1442062800000, 1442073600000, 0, 4, 2] + - ["NL", 1442073600000, 166, 12162, 1442066400000, 1442077200000, 0, 4, 2] + - ["NL", 1442077200000, 878, 12162, 1442070000000, 1442080800000, 0, 4, 2] + - ["NL", 1442080800000, 8947, 12162, 1442073600000, 1442084400000, 0, 4, 2] + - ["NL", 1442084400000, 436, 12162, 1442077200000, 1442088000000, 0, 4, 3] + - ["NL", 1442088000000, 12, 12162, 1442080800000, 1442091600000, 0, 4, 3] + - ["NL", 1442091600000, 19, 12162, 1442084400000, 1442095200000, 0, 4, 3] + - ["NL", 1442095200000, 70, 12162, 1442088000000, 1442098800000, 0, 4, 3] + - ["NL", 1442098800000, 4, 12162, 1442091600000, null, 0, 4, 3] + - ["NO", 1442019600000, 48, 432, null, 1442048400000, 48, 2, 1] + - ["NO", 1442048400000, -447, 432, null, 1442052000000, 48, 2, 1] + - ["NO", 1442052000000, 447, 432, 1442019600000, 1442055600000, 48, 2, 1] + - ["NO", 1442055600000, 29, 432, 1442048400000, 1442066400000, 48, 2, 1] + - ["NO", 1442066400000, 71, 432, 1442052000000, 1442073600000, 48, 2, 2] + - ["NO", 1442073600000, 222, 432, 1442055600000, 1442080800000, 48, 2, 2] + - ["NO", 1442080800000, 31, 432, 1442066400000, 1442088000000, 48, 2, 2] + - ["NO", 1442088000000, 15, 432, 1442073600000, 1442091600000, 48, 2, 2] + - ["NO", 1442091600000, 15, 432, 1442080800000, 1442095200000, 48, 2, 3] + - ["NO", 1442095200000, -1, 432, 1442088000000, 1442098800000, 48, 2, 3] + - ["NO", 1442098800000, 2, 432, 1442091600000, null, 48, 2, 3] + - ["NP", 1442048400000, 61, 61, null, null, 61, 61, 1] + - ["NZ", 1442019600000, 28, 1693, null, 1442026800000, 28, -2, 1] + - ["NZ", 1442026800000, 635, 1693, null, 1442037600000, 28, -2, 1] + - ["NZ", 1442037600000, 66, 1693, 1442019600000, 1442048400000, 28, -2, 1] + - ["NZ", 1442048400000, 189, 1693, 1442026800000, 1442059200000, 28, -2, 2] + - ["NZ", 1442059200000, 428, 1693, 1442037600000, 1442084400000, 28, -2, 2] + - ["NZ", 1442084400000, -52, 1693, 1442048400000, 1442088000000, 28, -2, 2] + - ["NZ", 1442088000000, 405, 1693, 1442059200000, 1442095200000, 28, -2, 3] + - ["NZ", 1442095200000, -4, 1693, 1442084400000, 1442098800000, 28, -2, 3] + - ["NZ", 1442098800000, -2, 1693, 1442088000000, null, 28, -2, 3] + - ["OM", 1442052000000, 0, 0, null, null, 0, 0, 1] + - ["PA", 1442026800000, 0, 0, null, null, 0, 0, 1] + - ["PE", 1442019600000, 523, 2134, null, 1442023200000, 523, 1861, 1] + - ["PE", 1442023200000, 26, 2134, null, 1442026800000, 523, 1861, 1] + - ["PE", 1442026800000, -12, 2134, 1442019600000, 1442062800000, 523, 1861, 1] + - ["PE", 1442062800000, -12, 2134, 1442023200000, 1442077200000, 523, 1861, 2] + - ["PE", 1442077200000, -163, 2134, 1442026800000, 1442080800000, 523, 1861, 2] + - ["PE", 1442080800000, -2, 2134, 1442062800000, 1442084400000, 523, 1861, 2] + - ["PE", 1442084400000, -68, 2134, 1442077200000, 1442095200000, 523, 1861, 3] + - ["PE", 1442095200000, -19, 2134, 1442080800000, 1442098800000, 523, 1861, 3] + - ["PE", 1442098800000, 1861, 2134, 1442084400000, null, 523, 1861, 3] + - ["PH", 1442019600000, 6, 6613, null, 1442023200000, 6, 8, 1] + - ["PH", 1442023200000, 459, 6613, null, 1442026800000, 6, 8, 1] + - ["PH", 1442026800000, 910, 6613, 1442019600000, 1442030400000, 6, 8, 1] + - ["PH", 1442030400000, 26, 6613, 1442023200000, 1442034000000, 6, 8, 1] + - ["PH", 1442034000000, 59, 6613, 1442026800000, 1442037600000, 6, 8, 1] + - ["PH", 1442037600000, 17, 6613, 1442030400000, 1442041200000, 6, 8, 1] + - ["PH", 1442041200000, 0, 6613, 1442034000000, 1442044800000, 6, 8, 1] + - ["PH", 1442044800000, 55, 6613, 1442037600000, 1442048400000, 6, 8, 2] + - ["PH", 1442048400000, 62, 6613, 1442041200000, 1442052000000, 6, 8, 2] + - ["PH", 1442052000000, 22, 6613, 1442044800000, 1442055600000, 6, 8, 2] + - ["PH", 1442055600000, 1969, 6613, 1442048400000, 1442059200000, 6, 8, 2] + - ["PH", 1442059200000, 273, 6613, 1442052000000, 1442062800000, 6, 8, 2] + - ["PH", 1442062800000, 171, 6613, 1442055600000, 1442066400000, 6, 8, 2] + - ["PH", 1442066400000, 1880, 6613, 1442059200000, 1442070000000, 6, 8, 2] + - ["PH", 1442070000000, 34, 6613, 1442062800000, 1442073600000, 6, 8, 3] + - ["PH", 1442073600000, -227, 6613, 1442066400000, 1442077200000, 6, 8, 3] + - ["PH", 1442077200000, 2, 6613, 1442070000000, 1442080800000, 6, 8, 3] + - ["PH", 1442080800000, 32, 6613, 1442073600000, 1442084400000, 6, 8, 3] + - ["PH", 1442084400000, 39, 6613, 1442077200000, 1442091600000, 6, 8, 3] + - ["PH", 1442091600000, 816, 6613, 1442080800000, 1442098800000, 6, 8, 3] + - ["PH", 1442098800000, 8, 6613, 1442084400000, null, 6, 8, 3] + - ["PK", 1442019600000, 335, 641, null, 1442026800000, 335, 43, 1] + - ["PK", 1442026800000, 101, 641, null, 1442037600000, 335, 43, 1] + - ["PK", 1442037600000, 100, 641, 1442019600000, 1442041200000, 335, 43, 1] + - ["PK", 1442041200000, 24, 641, 1442026800000, 1442048400000, 335, 43, 2] + - ["PK", 1442048400000, 15, 641, 1442037600000, 1442062800000, 335, 43, 2] + - ["PK", 1442062800000, 23, 641, 1442041200000, 1442070000000, 335, 43, 3] + - ["PK", 1442070000000, 43, 641, 1442048400000, null, 335, 43, 3] + - ["PL", 1442037600000, 95, 9815, null, 1442041200000, 95, -9, 1] + - ["PL", 1442041200000, 281, 9815, null, 1442044800000, 95, -9, 1] + - ["PL", 1442044800000, 319, 9815, 1442037600000, 1442048400000, 95, -9, 1] + - ["PL", 1442048400000, 366, 9815, 1442041200000, 1442052000000, 95, -9, 1] + - ["PL", 1442052000000, 330, 9815, 1442044800000, 1442055600000, 95, -9, 1] + - ["PL", 1442055600000, 410, 9815, 1442048400000, 1442059200000, 95, -9, 1] + - ["PL", 1442059200000, 199, 9815, 1442052000000, 1442062800000, 95, -9, 2] + - ["PL", 1442062800000, 4171, 9815, 1442055600000, 1442066400000, 95, -9, 2] + - ["PL", 1442066400000, 34, 9815, 1442059200000, 1442070000000, 95, -9, 2] + - ["PL", 1442070000000, 146, 9815, 1442062800000, 1442073600000, 95, -9, 2] + - ["PL", 1442073600000, 30, 9815, 1442066400000, 1442077200000, 95, -9, 2] + - ["PL", 1442077200000, 324, 9815, 1442070000000, 1442080800000, 95, -9, 2] + - ["PL", 1442080800000, 7, 9815, 1442073600000, 1442084400000, 95, -9, 3] + - ["PL", 1442084400000, 13, 9815, 1442077200000, 1442088000000, 95, -9, 3] + - ["PL", 1442088000000, 346, 9815, 1442080800000, 1442091600000, 95, -9, 3] + - ["PL", 1442091600000, 902, 9815, 1442084400000, 1442095200000, 95, -9, 3] + - ["PL", 1442095200000, 1851, 9815, 1442088000000, 1442098800000, 95, -9, 3] + - ["PL", 1442098800000, -9, 9815, 1442091600000, null, 95, -9, 3] + - ["PR", 1442026800000, 22, 23, null, 1442030400000, 22, 29, 1] + - ["PR", 1442030400000, 2, 23, null, 1442059200000, 22, 29, 1] + - ["PR", 1442059200000, -35, 23, 1442026800000, 1442077200000, 22, 29, 2] + - ["PR", 1442077200000, 5, 23, 1442030400000, 1442095200000, 22, 29, 2] + - ["PR", 1442095200000, 29, 23, 1442059200000, null, 22, 29, 3] + - ["PT", 1442019600000, 172, 4037, null, 1442044800000, 172, 2, 1] + - ["PT", 1442044800000, 11, 4037, null, 1442052000000, 172, 2, 1] + - ["PT", 1442052000000, 102, 4037, 1442019600000, 1442066400000, 172, 2, 1] + - ["PT", 1442066400000, 12, 4037, 1442044800000, 1442070000000, 172, 2, 1] + - ["PT", 1442070000000, 3470, 4037, 1442052000000, 1442077200000, 172, 2, 2] + - ["PT", 1442077200000, -75, 4037, 1442066400000, 1442080800000, 172, 2, 2] + - ["PT", 1442080800000, -79, 4037, 1442070000000, 1442088000000, 172, 2, 2] + - ["PT", 1442088000000, 403, 4037, 1442077200000, 1442095200000, 172, 2, 3] + - ["PT", 1442095200000, 19, 4037, 1442080800000, 1442098800000, 172, 2, 3] + - ["PT", 1442098800000, 2, 4037, 1442088000000, null, 172, 2, 3] + - ["PY", 1442019600000, 1, 634, null, 1442080800000, 1, 628, 1] + - ["PY", 1442080800000, 5, 634, null, 1442084400000, 1, 628, 2] + - ["PY", 1442084400000, 628, 634, 1442019600000, null, 1, 628, 3] + - ["QA", 1442041200000, 13, 13, null, null, 13, 13, 1] + - ["RO", 1442034000000, 68, 2893, null, 1442041200000, 68, 824, 1] + - ["RO", 1442041200000, 845, 2893, null, 1442044800000, 68, 824, 1] + - ["RO", 1442044800000, 284, 2893, 1442034000000, 1442052000000, 68, 824, 1] + - ["RO", 1442052000000, 319, 2893, 1442041200000, 1442055600000, 68, 824, 1] + - ["RO", 1442055600000, 26, 2893, 1442044800000, 1442062800000, 68, 824, 2] + - ["RO", 1442062800000, 541, 2893, 1442052000000, 1442070000000, 68, 824, 2] + - ["RO", 1442070000000, -29, 2893, 1442055600000, 1442073600000, 68, 824, 2] + - ["RO", 1442073600000, 15, 2893, 1442062800000, 1442091600000, 68, 824, 3] + - ["RO", 1442091600000, 0, 2893, 1442070000000, 1442095200000, 68, 824, 3] + - ["RO", 1442095200000, 824, 2893, 1442073600000, null, 68, 824, 3] + - ["RS", 1442019600000, 6, 906, null, 1442062800000, 6, -15, 1] + - ["RS", 1442062800000, 13, 906, null, 1442066400000, 6, -15, 1] + - ["RS", 1442066400000, 0, 906, 1442019600000, 1442073600000, 6, -15, 1] + - ["RS", 1442073600000, 813, 906, 1442062800000, 1442080800000, 6, -15, 2] + - ["RS", 1442080800000, 0, 906, 1442066400000, 1442084400000, 6, -15, 2] + - ["RS", 1442084400000, 89, 906, 1442073600000, 1442091600000, 6, -15, 3] + - ["RS", 1442091600000, -15, 906, 1442080800000, null, 6, -15, 3] + - ["RU", 1442019600000, 2214, 48104, null, 1442023200000, 2214, 12098, 1] + - ["RU", 1442023200000, 299, 48104, null, 1442026800000, 2214, 12098, 1] + - ["RU", 1442026800000, 0, 48104, 1442019600000, 1442030400000, 2214, 12098, 1] + - ["RU", 1442030400000, 76, 48104, 1442023200000, 1442034000000, 2214, 12098, 1] + - ["RU", 1442034000000, 658, 48104, 1442026800000, 1442037600000, 2214, 12098, 1] + - ["RU", 1442037600000, -324, 48104, 1442030400000, 1442041200000, 2214, 12098, 1] + - ["RU", 1442041200000, 580, 48104, 1442034000000, 1442044800000, 2214, 12098, 1] + - ["RU", 1442044800000, 2564, 48104, 1442037600000, 1442048400000, 2214, 12098, 1] + - ["RU", 1442048400000, 1027, 48104, 1442041200000, 1442052000000, 2214, 12098, 2] + - ["RU", 1442052000000, 1214, 48104, 1442044800000, 1442055600000, 2214, 12098, 2] + - ["RU", 1442055600000, 499, 48104, 1442048400000, 1442059200000, 2214, 12098, 2] + - ["RU", 1442059200000, 3902, 48104, 1442052000000, 1442062800000, 2214, 12098, 2] + - ["RU", 1442062800000, 168, 48104, 1442055600000, 1442066400000, 2214, 12098, 2] + - ["RU", 1442066400000, 2047, 48104, 1442059200000, 1442070000000, 2214, 12098, 2] + - ["RU", 1442070000000, 4706, 48104, 1442062800000, 1442073600000, 2214, 12098, 2] + - ["RU", 1442073600000, 1618, 48104, 1442066400000, 1442077200000, 2214, 12098, 2] + - ["RU", 1442077200000, 1162, 48104, 1442070000000, 1442080800000, 2214, 12098, 3] + - ["RU", 1442080800000, 655, 48104, 1442073600000, 1442084400000, 2214, 12098, 3] + - ["RU", 1442084400000, 6461, 48104, 1442077200000, 1442088000000, 2214, 12098, 3] + - ["RU", 1442088000000, 2596, 48104, 1442080800000, 1442091600000, 2214, 12098, 3] + - ["RU", 1442091600000, 3449, 48104, 1442084400000, 1442095200000, 2214, 12098, 3] + - ["RU", 1442095200000, 435, 48104, 1442088000000, 1442098800000, 2214, 12098, 3] + - ["RU", 1442098800000, 12098, 48104, 1442091600000, null, 2214, 12098, 3] + - ["SA", 1442037600000, -97, 1614, null, 1442048400000, -97, 458, 1] + - ["SA", 1442048400000, 14, 1614, null, 1442055600000, -97, 458, 1] + - ["SA", 1442055600000, 11, 1614, 1442037600000, 1442059200000, -97, 458, 1] + - ["SA", 1442059200000, 0, 1614, 1442048400000, 1442066400000, -97, 458, 2] + - ["SA", 1442066400000, 1276, 1614, 1442055600000, 1442073600000, -97, 458, 2] + - ["SA", 1442073600000, 2, 1614, 1442059200000, 1442077200000, -97, 458, 2] + - ["SA", 1442077200000, -50, 1614, 1442066400000, 1442084400000, -97, 458, 3] + - ["SA", 1442084400000, 458, 1614, 1442073600000, null, -97, 458, 3] + - ["SE", 1442019600000, 109, 1838, null, 1442023200000, 109, 0, 1] + - ["SE", 1442023200000, 3, 1838, null, 1442030400000, 109, 0, 1] + - ["SE", 1442030400000, 30, 1838, 1442019600000, 1442041200000, 109, 0, 1] + - ["SE", 1442041200000, 91, 1838, 1442023200000, 1442048400000, 109, 0, 1] + - ["SE", 1442048400000, -145, 1838, 1442030400000, 1442052000000, 109, 0, 1] + - ["SE", 1442052000000, 1, 1838, 1442041200000, 1442055600000, 109, 0, 2] + - ["SE", 1442055600000, -5, 1838, 1442048400000, 1442059200000, 109, 0, 2] + - ["SE", 1442059200000, 1476, 1838, 1442052000000, 1442066400000, 109, 0, 2] + - ["SE", 1442066400000, 14, 1838, 1442055600000, 1442070000000, 109, 0, 2] + - ["SE", 1442070000000, 78, 1838, 1442059200000, 1442080800000, 109, 0, 2] + - ["SE", 1442080800000, 89, 1838, 1442066400000, 1442084400000, 109, 0, 3] + - ["SE", 1442084400000, 37, 1838, 1442070000000, 1442091600000, 109, 0, 3] + - ["SE", 1442091600000, -1, 1838, 1442080800000, 1442095200000, 109, 0, 3] + - ["SE", 1442095200000, 61, 1838, 1442084400000, 1442098800000, 109, 0, 3] + - ["SE", 1442098800000, 0, 1838, 1442091600000, null, 109, 0, 3] + - ["SG", 1442026800000, 2758, 3338, null, 1442030400000, 2758, 0, 1] + - ["SG", 1442030400000, 1, 3338, null, 1442037600000, 2758, 0, 1] + - ["SG", 1442037600000, 3, 3338, 1442026800000, 1442041200000, 2758, 0, 1] + - ["SG", 1442041200000, 59, 3338, 1442030400000, 1442044800000, 2758, 0, 2] + - ["SG", 1442044800000, 77, 3338, 1442037600000, 1442048400000, 2758, 0, 2] + - ["SG", 1442048400000, 52, 3338, 1442041200000, 1442062800000, 2758, 0, 2] + - ["SG", 1442062800000, 388, 3338, 1442044800000, 1442066400000, 2758, 0, 3] + - ["SG", 1442066400000, 0, 3338, 1442048400000, null, 2758, 0, 3] + - ["SI", 1442080800000, -45, -36, null, 1442091600000, -45, 9, 1] + - ["SI", 1442091600000, 9, -36, null, null, -45, 9, 2] + - ["SK", 1442037600000, -1, 379, null, 1442052000000, -1, 7, 1] + - ["SK", 1442052000000, 13, 379, null, 1442062800000, -1, 7, 1] + - ["SK", 1442062800000, 6, 379, 1442037600000, 1442073600000, -1, 7, 2] + - ["SK", 1442073600000, 446, 379, 1442052000000, 1442084400000, -1, 7, 2] + - ["SK", 1442084400000, -92, 379, 1442062800000, 1442098800000, -1, 7, 3] + - ["SK", 1442098800000, 7, 379, 1442073600000, null, -1, 7, 3] + - ["SV", 1442019600000, -1, 114, null, 1442084400000, -1, 9, 1] + - ["SV", 1442084400000, 106, 114, null, 1442088000000, -1, 9, 2] + - ["SV", 1442088000000, 9, 114, 1442019600000, null, -1, 9, 3] + - ["TH", 1442034000000, 0, 24, null, 1442041200000, 0, 13, 1] + - ["TH", 1442041200000, 3, 24, null, 1442044800000, 0, 13, 1] + - ["TH", 1442044800000, 110, 24, 1442034000000, 1442052000000, 0, 13, 1] + - ["TH", 1442052000000, -22, 24, 1442041200000, 1442055600000, 0, 13, 2] + - ["TH", 1442055600000, 0, 24, 1442044800000, 1442062800000, 0, 13, 2] + - ["TH", 1442062800000, -46, 24, 1442052000000, 1442066400000, 0, 13, 2] + - ["TH", 1442066400000, -34, 24, 1442055600000, 1442070000000, 0, 13, 3] + - ["TH", 1442070000000, 0, 24, 1442062800000, 1442084400000, 0, 13, 3] + - ["TH", 1442084400000, 13, 24, 1442066400000, null, 0, 13, 3] + - ["TJ", 1442048400000, 1471, 1471, null, null, 1471, 1471, 1] + - ["TN", 1442098800000, -9, -9, null, null, -9, -9, 1] + - ["TR", 1442023200000, 306, 7078, null, 1442041200000, 306, -29, 1] + - ["TR", 1442041200000, 1, 7078, null, 1442044800000, 306, -29, 1] + - ["TR", 1442044800000, 41, 7078, 1442023200000, 1442048400000, 306, -29, 1] + - ["TR", 1442048400000, 88, 7078, 1442041200000, 1442052000000, 306, -29, 1] + - ["TR", 1442052000000, 41, 7078, 1442044800000, 1442055600000, 306, -29, 1] + - ["TR", 1442055600000, 299, 7078, 1442048400000, 1442062800000, 306, -29, 2] + - ["TR", 1442062800000, 315, 7078, 1442052000000, 1442066400000, 306, -29, 2] + - ["TR", 1442066400000, 85, 7078, 1442055600000, 1442070000000, 306, -29, 2] + - ["TR", 1442070000000, 236, 7078, 1442062800000, 1442077200000, 306, -29, 2] + - ["TR", 1442077200000, 89, 7078, 1442066400000, 1442080800000, 306, -29, 2] + - ["TR", 1442080800000, -1, 7078, 1442070000000, 1442084400000, 306, -29, 3] + - ["TR", 1442084400000, 170, 7078, 1442077200000, 1442088000000, 306, -29, 3] + - ["TR", 1442088000000, 2389, 7078, 1442080800000, 1442091600000, 306, -29, 3] + - ["TR", 1442091600000, 3048, 7078, 1442084400000, 1442095200000, 306, -29, 3] + - ["TR", 1442095200000, -29, 7078, 1442088000000, null, 306, -29, 3] + - ["TT", 1442088000000, 9, 9, null, null, 9, 9, 1] + - ["TW", 1442016000000, 92, 3656, null, 1442019600000, 92, -60, 1] + - ["TW", 1442019600000, 0, 3656, null, 1442023200000, 92, -60, 1] + - ["TW", 1442023200000, 97, 3656, 1442016000000, 1442026800000, 92, -60, 1] + - ["TW", 1442026800000, 680, 3656, 1442019600000, 1442030400000, 92, -60, 1] + - ["TW", 1442030400000, 0, 3656, 1442023200000, 1442034000000, 92, -60, 1] + - ["TW", 1442034000000, 143, 3656, 1442026800000, 1442037600000, 92, -60, 1] + - ["TW", 1442037600000, 266, 3656, 1442030400000, 1442041200000, 92, -60, 1] + - ["TW", 1442041200000, 366, 3656, 1442034000000, 1442044800000, 92, -60, 1] + - ["TW", 1442044800000, 24, 3656, 1442037600000, 1442048400000, 92, -60, 2] + - ["TW", 1442048400000, 75, 3656, 1442041200000, 1442052000000, 92, -60, 2] + - ["TW", 1442052000000, 24, 3656, 1442044800000, 1442055600000, 92, -60, 2] + - ["TW", 1442055600000, 48, 3656, 1442048400000, 1442059200000, 92, -60, 2] + - ["TW", 1442059200000, -157, 3656, 1442052000000, 1442062800000, 92, -60, 2] + - ["TW", 1442062800000, -272, 3656, 1442055600000, 1442066400000, 92, -60, 2] + - ["TW", 1442066400000, 624, 3656, 1442059200000, 1442070000000, 92, -60, 2] + - ["TW", 1442070000000, 485, 3656, 1442062800000, 1442073600000, 92, -60, 3] + - ["TW", 1442073600000, 772, 3656, 1442066400000, 1442077200000, 92, -60, 3] + - ["TW", 1442077200000, 502, 3656, 1442070000000, 1442080800000, 92, -60, 3] + - ["TW", 1442080800000, 24, 3656, 1442073600000, 1442084400000, 92, -60, 3] + - ["TW", 1442084400000, 0, 3656, 1442077200000, 1442095200000, 92, -60, 3] + - ["TW", 1442095200000, -77, 3656, 1442080800000, 1442098800000, 92, -60, 3] + - ["TW", 1442098800000, -60, 3656, 1442084400000, null, 92, -60, 3] + - ["UA", 1442034000000, 3468, 24898, null, 1442037600000, 3468, 38, 1] + - ["UA", 1442037600000, -1, 24898, null, 1442041200000, 3468, 38, 1] + - ["UA", 1442041200000, 74, 24898, 1442034000000, 1442044800000, 3468, 38, 1] + - ["UA", 1442044800000, 280, 24898, 1442037600000, 1442048400000, 3468, 38, 1] + - ["UA", 1442048400000, 2, 24898, 1442041200000, 1442052000000, 3468, 38, 1] + - ["UA", 1442052000000, 410, 24898, 1442044800000, 1442055600000, 3468, 38, 1] + - ["UA", 1442055600000, 14202, 24898, 1442048400000, 1442059200000, 3468, 38, 1] + - ["UA", 1442059200000, -2, 24898, 1442052000000, 1442062800000, 3468, 38, 2] + - ["UA", 1442062800000, 773, 24898, 1442055600000, 1442066400000, 3468, 38, 2] + - ["UA", 1442066400000, 296, 24898, 1442059200000, 1442070000000, 3468, 38, 2] + - ["UA", 1442070000000, 1733, 24898, 1442062800000, 1442073600000, 3468, 38, 2] + - ["UA", 1442073600000, 4241, 24898, 1442066400000, 1442077200000, 3468, 38, 2] + - ["UA", 1442077200000, -181, 24898, 1442070000000, 1442080800000, 3468, 38, 2] + - ["UA", 1442080800000, -1, 24898, 1442073600000, 1442084400000, 3468, 38, 3] + - ["UA", 1442084400000, 5, 24898, 1442077200000, 1442088000000, 3468, 38, 3] + - ["UA", 1442088000000, -21, 24898, 1442080800000, 1442091600000, 3468, 38, 3] + - ["UA", 1442091600000, -388, 24898, 1442084400000, 1442095200000, 3468, 38, 3] + - ["UA", 1442095200000, -30, 24898, 1442088000000, 1442098800000, 3468, 38, 3] + - ["UA", 1442098800000, 38, 24898, 1442091600000, null, 3468, 38, 3] + - ["UG", 1442070000000, 1, 1, null, null, 1, 1, 1] + - ["US", 1442016000000, 0, 38882, null, 1442019600000, 0, 3575, 1] + - ["US", 1442019600000, 1043, 38882, null, 1442023200000, 0, 3575, 1] + - ["US", 1442023200000, 2844, 38882, 1442016000000, 1442026800000, 0, 3575, 1] + - ["US", 1442026800000, 1512, 38882, 1442019600000, 1442030400000, 0, 3575, 1] + - ["US", 1442030400000, 2023, 38882, 1442023200000, 1442034000000, 0, 3575, 1] + - ["US", 1442034000000, 3648, 38882, 1442026800000, 1442037600000, 0, 3575, 1] + - ["US", 1442037600000, 3675, 38882, 1442030400000, 1442041200000, 0, 3575, 1] + - ["US", 1442041200000, 1999, 38882, 1442034000000, 1442044800000, 0, 3575, 1] + - ["US", 1442044800000, 139, 38882, 1442037600000, 1442048400000, 0, 3575, 2] + - ["US", 1442048400000, -466, 38882, 1442041200000, 1442052000000, 0, 3575, 2] + - ["US", 1442052000000, -2, 38882, 1442044800000, 1442055600000, 0, 3575, 2] + - ["US", 1442055600000, 156, 38882, 1442048400000, 1442059200000, 0, 3575, 2] + - ["US", 1442059200000, 11, 38882, 1442052000000, 1442062800000, 0, 3575, 2] + - ["US", 1442062800000, 47, 38882, 1442055600000, 1442066400000, 0, 3575, 2] + - ["US", 1442066400000, 772, 38882, 1442059200000, 1442070000000, 0, 3575, 2] + - ["US", 1442070000000, 3505, 38882, 1442062800000, 1442073600000, 0, 3575, 2] + - ["US", 1442073600000, 1100, 38882, 1442066400000, 1442077200000, 0, 3575, 3] + - ["US", 1442077200000, 2168, 38882, 1442070000000, 1442080800000, 0, 3575, 3] + - ["US", 1442080800000, 4001, 38882, 1442073600000, 1442084400000, 0, 3575, 3] + - ["US", 1442084400000, 2523, 38882, 1442077200000, 1442088000000, 0, 3575, 3] + - ["US", 1442088000000, 1691, 38882, 1442080800000, 1442091600000, 0, 3575, 3] + - ["US", 1442091600000, 2502, 38882, 1442084400000, 1442095200000, 0, 3575, 3] + - ["US", 1442095200000, 416, 38882, 1442088000000, 1442098800000, 0, 3575, 3] + - ["US", 1442098800000, 3575, 38882, 1442091600000, null, 0, 3575, 3] + - ["UY", 1442019600000, 77, 936, null, 1442023200000, 77, 23, 1] + - ["UY", 1442023200000, 517, 936, null, 1442026800000, 77, 23, 1] + - ["UY", 1442026800000, 76, 936, 1442019600000, 1442037600000, 77, 23, 1] + - ["UY", 1442037600000, 1, 936, 1442023200000, 1442070000000, 77, 23, 2] + - ["UY", 1442070000000, 284, 936, 1442026800000, 1442073600000, 77, 23, 2] + - ["UY", 1442073600000, -42, 936, 1442037600000, 1442077200000, 77, 23, 3] + - ["UY", 1442077200000, 23, 936, 1442070000000, null, 77, 23, 3] + - ["UZ", 1442044800000, 1369, 1369, null, null, 1369, 1369, 1] + - ["VE", 1442023200000, 115, 1101, null, 1442026800000, 115, 9, 1] + - ["VE", 1442026800000, -17, 1101, null, 1442030400000, 115, 9, 1] + - ["VE", 1442030400000, 51, 1101, 1442023200000, 1442034000000, 115, 9, 1] + - ["VE", 1442034000000, -2, 1101, 1442026800000, 1442066400000, 115, 9, 1] + - ["VE", 1442066400000, 18, 1101, 1442030400000, 1442070000000, 115, 9, 2] + - ["VE", 1442070000000, 420, 1101, 1442034000000, 1442077200000, 115, 9, 2] + - ["VE", 1442077200000, 412, 1101, 1442066400000, 1442084400000, 115, 9, 2] + - ["VE", 1442084400000, 60, 1101, 1442070000000, 1442095200000, 115, 9, 3] + - ["VE", 1442095200000, 35, 1101, 1442077200000, 1442098800000, 115, 9, 3] + - ["VE", 1442098800000, 9, 1101, 1442084400000, null, 115, 9, 3] + - ["VG", 1442062800000, -238, -238, null, null, -238, -238, 1] + - ["VN", 1442023200000, -9, 1560, null, 1442026800000, -9, -10, 1] + - ["VN", 1442026800000, 63, 1560, null, 1442034000000, -9, -10, 1] + - ["VN", 1442034000000, -29, 1560, 1442023200000, 1442037600000, -9, -10, 1] + - ["VN", 1442037600000, -11, 1560, 1442026800000, 1442041200000, -9, -10, 1] + - ["VN", 1442041200000, 0, 1560, 1442034000000, 1442048400000, -9, -10, 1] + - ["VN", 1442048400000, -15, 1560, 1442037600000, 1442052000000, -9, -10, 2] + - ["VN", 1442052000000, 90, 1560, 1442041200000, 1442055600000, -9, -10, 2] + - ["VN", 1442055600000, 37, 1560, 1442048400000, 1442059200000, -9, -10, 2] + - ["VN", 1442059200000, 8, 1560, 1442052000000, 1442062800000, -9, -10, 2] + - ["VN", 1442062800000, 146, 1560, 1442055600000, 1442066400000, -9, -10, 3] + - ["VN", 1442066400000, 811, 1560, 1442059200000, 1442070000000, -9, -10, 3] + - ["VN", 1442070000000, 479, 1560, 1442062800000, 1442084400000, -9, -10, 3] + - ["VN", 1442084400000, -10, 1560, 1442066400000, null, -9, -10, 3] + - ["ZA", 1442034000000, -3, 127, null, 1442048400000, -3, 1, 1] + - ["ZA", 1442048400000, 79, 127, null, 1442059200000, -3, 1, 1] + - ["ZA", 1442059200000, 50, 127, 1442034000000, 1442070000000, -3, 1, 2] + - ["ZA", 1442070000000, 0, 127, 1442048400000, 1442091600000, -3, 1, 2] + - ["ZA", 1442091600000, 1, 127, 1442059200000, null, -3, 1, 3] + - ["ZM", 1442041200000, 133, 133, null, null, 133, 133, 1] + - ["ZW", 1442044800000, 0, 254, null, 1442048400000, 0, 254, 1] + - ["ZW", 1442048400000, 254, 254, null, null, 0, 254, 2] \ No newline at end of file