From 8440c8a140ccb3c2cb6314685579d3c41a115896 Mon Sep 17 00:00:00 2001 From: saydakov Date: Tue, 24 Oct 2017 12:25:22 -0700 Subject: [PATCH 01/17] numeric quantiles sketch aggregator --- extensions-core/datasketches/pom.xml | 2 +- ...mbiningDoublesSketchAggregatorFactory.java | 63 +++ .../quantiles/DoublesSketchAggregator.java | 80 ++++ .../DoublesSketchAggregatorFactory.java | 264 +++++++++++ .../DoublesSketchBufferAggregator.java | 144 ++++++ .../DoublesSketchComplexMetricSerde.java | 98 ++++ .../DoublesSketchDoubleAggregator.java | 78 ++++ .../DoublesSketchDoubleBufferAggregator.java | 138 ++++++ .../DoublesSketchHistogramPostAggregator.java | 160 +++++++ .../DoublesSketchJsonSerializer.java | 40 ++ .../quantiles/DoublesSketchModule.java | 65 +++ .../DoublesSketchObjectStrategy.java | 64 +++ .../quantiles/DoublesSketchOperations.java | 57 +++ .../DoublesSketchQuantilePostAggregator.java | 149 ++++++ .../DoublesSketchQuantilesPostAggregator.java | 155 +++++++ .../DoublesSketchToStringPostAggregator.java | 138 ++++++ .../EmptyDoublesSketchAggregator.java | 60 +++ .../EmptyDoublesSketchBufferAggregator.java | 68 +++ .../datasketches/theta/SynchronizedUnion.java | 10 +- .../io.druid.initialization.DruidModule | 1 + .../DoublesSketchAggregatorTest.java | 423 ++++++++++++++++++ .../quantiles/GenerateTestData.java | 71 +++ .../quantiles/doubles_build_data.tsv | 400 +++++++++++++++++ .../quantiles/doubles_sketch_data.tsv | 20 + 24 files changed, 2746 insertions(+), 2 deletions(-) create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchBufferAggregator.java create mode 100644 extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java create mode 100644 extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java create mode 100644 extensions-core/datasketches/src/test/resources/quantiles/doubles_build_data.tsv create mode 100644 extensions-core/datasketches/src/test/resources/quantiles/doubles_sketch_data.tsv diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index c037f8317456..974a19fcb430 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -38,7 +38,7 @@ com.yahoo.datasketches sketches-core - 0.10.1 + 0.10.2 io.druid diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java new file mode 100644 index 000000000000..841abf9759bc --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.sketches.quantiles.DoublesSketch; +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.BufferAggregator; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; + +public class CombiningDoublesSketchAggregatorFactory extends DoublesSketchAggregatorFactory +{ + + private static final byte CACHE_TYPE_ID = 111; + + @JsonCreator + public CombiningDoublesSketchAggregatorFactory( + @JsonProperty("name") final String name, + @JsonProperty("k") final Integer k) + { + super(name, name, k, CACHE_TYPE_ID); + } + + @Override + public Aggregator factorize(final ColumnSelectorFactory metricFactory) + { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); + if (selector == null) { + return new EmptyDoublesSketchAggregator(); + } + return new DoublesSketchAggregator(selector, getK()); + } + + @Override + public BufferAggregator factorizeBuffered(final ColumnSelectorFactory metricFactory) + { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); + if (selector == null) { + return new EmptyDoublesSketchBufferAggregator(); + } + return new DoublesSketchBufferAggregator(selector, getK(), getMaxIntermediateSize()); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java new file mode 100644 index 000000000000..553b49abf25e --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java @@ -0,0 +1,80 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.DoublesUnion; + +import io.druid.query.aggregation.Aggregator; +import io.druid.segment.ColumnValueSelector; + +public class DoublesSketchAggregator implements Aggregator +{ + + private final ColumnValueSelector selector; + private DoublesUnion union; + + public DoublesSketchAggregator(final ColumnValueSelector selector, final int k) + { + this.selector = selector; + union = DoublesUnion.builder().setMaxK(k).build(); + } + + @Override + public synchronized void aggregate() + { + final DoublesSketch sketch = selector.getObject(); + if (sketch == null) { + return; + } + union.update(sketch); + } + + @Override + public Object get() + { + return union.getResult(); + } + + @Override + public float getFloat() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void reset() + { + union.reset(); + } + + @Override + public void close() + { + union = null; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java new file mode 100644 index 000000000000..ab5c35f9a87c --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -0,0 +1,264 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.metamx.common.IAE; +import com.yahoo.sketches.Util; +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.DoublesUnion; + +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorFactoryNotMergeableException; +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.cache.CacheKeyBuilder; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; +import io.druid.segment.column.ValueType; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class DoublesSketchAggregatorFactory extends AggregatorFactory +{ + + private static final int DEFAULT_K = 128; + private static final byte CACHE_TYPE_ID = 110; + + // Used for sketch size estimation. + private static final long MAX_STREAM_LENGTH = 1_000_000_000; + + private final String name; + private final String fieldName; + private final int k; + private final byte cacheTypeId; + + @JsonCreator + public DoublesSketchAggregatorFactory( + @JsonProperty("name") final String name, + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("k") final Integer k) + { + this(name, fieldName, k, CACHE_TYPE_ID); + } + + DoublesSketchAggregatorFactory(final String name, final String fieldName, final Integer k, final byte cacheTypeId) + { + if (name == null) { + throw new IAE("Must have a valid, non-null aggregator name"); + } + this.name = name; + if (fieldName == null) { + throw new IAE("Parameter fieldName must be specified"); + } + this.fieldName = fieldName; + this.k = k == null ? DEFAULT_K : k; + Util.checkIfPowerOf2(this.k, "size"); + this.cacheTypeId = cacheTypeId; + } + + @Override + public Aggregator factorize(final ColumnSelectorFactory metricFactory) + { + if (metricFactory.getColumnCapabilities(fieldName) != null + && ValueType.isNumeric(metricFactory.getColumnCapabilities(fieldName).getType())) { + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + if (valueSelector == null) { + return new EmptyDoublesSketchAggregator(); + } + return new DoublesSketchDoubleAggregator(valueSelector, k); + } + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector == null) { + return new EmptyDoublesSketchAggregator(); + } + return new DoublesSketchAggregator(selector, k); + } + + @Override + public BufferAggregator factorizeBuffered(final ColumnSelectorFactory metricFactory) + { + if (metricFactory.getColumnCapabilities(fieldName) != null + && ValueType.isNumeric(metricFactory.getColumnCapabilities(fieldName).getType())) { + return getDoubleBufferAggregator(metricFactory.makeColumnValueSelector(fieldName)); + } + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector == null) { + return new EmptyDoublesSketchBufferAggregator(); + } + return new DoublesSketchBufferAggregator(selector, k, getMaxIntermediateSize()); + } + + private BufferAggregator getDoubleBufferAggregator(final ColumnValueSelector selector) + { + if (selector == null) { + return new EmptyDoublesSketchBufferAggregator(); + } + return new DoublesSketchDoubleBufferAggregator(selector, k, getMaxIntermediateSize()); + } + + @Override + public Object deserialize(final Object object) + { + return DoublesSketchOperations.deserialize(object); + } + + public static final Comparator COMPARATOR = new Comparator() + { + @Override + public int compare(DoublesSketch a, DoublesSketch b) + { + return Long.compare(a.getN(), b.getN()); + } + }; + + @Override + public Comparator getComparator() + { + return COMPARATOR; + } + + @Override + public Object combine(final Object lhs, final Object rhs) + { + final DoublesUnion union = DoublesUnion.builder().setMaxK(k).build(); + union.update((DoublesSketch) lhs); + union.update((DoublesSketch) rhs); + return union.getResultAndReset(); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public String getFieldName() + { + return fieldName; + } + + @JsonProperty + public int getK() + { + return k; + } + + @Override + public List requiredFields() + { + return Collections.singletonList(fieldName); + } + + @Override + public int getMaxIntermediateSize() + { + return DoublesSketch.getUpdatableStorageBytes(k, MAX_STREAM_LENGTH); + } + + @Override + public List getRequiredColumns() + { + return Collections. singletonList( + new DoublesSketchAggregatorFactory( + fieldName, + fieldName, + k)); + } + + @Override + public AggregatorFactory getCombiningFactory() + { + return new CombiningDoublesSketchAggregatorFactory(name, k); + } + + @Override + public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException + { + if (other.getName().equals(this.getName()) && other instanceof DoublesSketchAggregatorFactory) { + return new CombiningDoublesSketchAggregatorFactory(name, k); + } else { + throw new AggregatorFactoryNotMergeableException(this, other); + } + } + + @Override + public Object finalizeComputation(final Object object) + { + return ((DoublesSketch) object).getN(); + } + + @Override + public String getTypeName() + { + return DoublesSketchModule.DOUBLES_SKETCH; + } + + @Override + public byte[] getCacheKey() + { + return new CacheKeyBuilder(cacheTypeId).appendString(name).appendString(fieldName).appendInt(k).build(); + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || !getClass().equals(o.getClass())) { + return false; + } + final DoublesSketchAggregatorFactory that = (DoublesSketchAggregatorFactory) o; + if (!fieldName.equals(that.fieldName)) { + return false; + } + if (k != that.k) { + return false; + } + return true; + } + + @Override + public int hashCode() + { + int result = name.hashCode(); + result = 31 * result + fieldName.hashCode(); + result = 31 * result + Integer.hashCode(k); + result = 31 * result + Byte.hashCode(cacheTypeId); + return result; + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "name=" + name + + ", fieldName=" + fieldName + + ", k=" + k + + "}"; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java new file mode 100644 index 000000000000..20e0ab5a5599 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java @@ -0,0 +1,144 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.yahoo.memory.WritableMemory; +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.DoublesUnion; +import com.yahoo.sketches.quantiles.DoublesUnionBuilder; +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.ColumnValueSelector; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; + +import java.nio.ByteBuffer; +import java.util.IdentityHashMap; + +public class DoublesSketchBufferAggregator implements BufferAggregator +{ + + private final ColumnValueSelector selector; + private final int k; + private final int maxIntermediateSize; + private final IdentityHashMap memCache = new IdentityHashMap<>(); + private final IdentityHashMap> unions = new IdentityHashMap<>(); + + public DoublesSketchBufferAggregator( + final ColumnValueSelector selector, + final int k, + final int maxIntermediateSize) + { + this.selector = selector; + this.k = k; + this.maxIntermediateSize = maxIntermediateSize; + } + + @Override + public void init(final ByteBuffer buffer, final int position) + { + final WritableMemory mem = getMemory(buffer); + final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); + final DoublesUnion union = DoublesUnion.builder().setMaxK(k).build(region); + putUnion(buffer, position, union); + } + + @Override + public synchronized void aggregate(final ByteBuffer buffer, final int position) + { + final DoublesSketch sketch = selector.getObject(); + if (sketch == null) { + return; + } + final DoublesUnion union = unions.get(buffer).get(position); + union.update(sketch); + } + + @Override + public Object get(final ByteBuffer buffer, final int position) + { + return unions.get(buffer).get(position).getResult(); + } + + @Override + public float getFloat(final ByteBuffer buffer, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong(final ByteBuffer buffer, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() + { + unions.clear(); + memCache.clear(); + } + + @Override + public void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) + { + DoublesUnion union = unions.get(oldBuffer).get(oldPosition); + final WritableMemory oldMem = getMemory(oldBuffer).writableRegion(oldPosition, maxIntermediateSize); + if (union.isSameResource(oldMem)) { // union was not relocated on heap + final WritableMemory newMem = getMemory(newBuffer).writableRegion(newPosition, maxIntermediateSize); + union = DoublesUnionBuilder.wrap(newMem); + } + putUnion(newBuffer, newPosition, union); + + Int2ObjectMap map = unions.get(oldBuffer); + map.remove(oldPosition); + if (map.isEmpty()) { + unions.remove(oldBuffer); + memCache.remove(oldBuffer); + } + } + + private WritableMemory getMemory(final ByteBuffer buffer) + { + WritableMemory mem = memCache.get(buffer); + if (mem == null) { + mem = WritableMemory.wrap(buffer); + memCache.put(buffer, mem); + } + return mem; + } + + private void putUnion(final ByteBuffer buffer, final int position, final DoublesUnion union) + { + Int2ObjectMap map = unions.get(buffer); + if (map == null) { + map = new Int2ObjectOpenHashMap<>(); + unions.put(buffer, map); + } + map.put(position, union); + } + + @Override + public void inspectRuntimeShape(final RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java new file mode 100644 index 000000000000..da81f135923c --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java @@ -0,0 +1,98 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.yahoo.memory.Memory; +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.UpdateDoublesSketch; +import io.druid.data.input.InputRow; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.ObjectStrategy; +import io.druid.segment.serde.ComplexColumnPartSupplier; +import io.druid.segment.serde.ComplexMetricExtractor; +import io.druid.segment.serde.ComplexMetricSerde; + +import java.nio.ByteBuffer; + +public class DoublesSketchComplexMetricSerde extends ComplexMetricSerde +{ + + private static final DoublesSketchObjectStrategy strategy = new DoublesSketchObjectStrategy(); + + @Override + public String getTypeName() + { + return DoublesSketchModule.DOUBLES_SKETCH; + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return strategy; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + return new ComplexMetricExtractor() + { + @Override + public Class extractedClass() + { + return DoublesSketch.class; + } + + @Override + public Object extractValue(final InputRow inputRow, final String metricName) + { + final Object object = inputRow.getRaw(metricName); + if (object instanceof String) { + String objectString = (String) object; + if (Character.isDigit((objectString).charAt(0))) { + try { + Double doubleValue = Double.parseDouble(objectString); + UpdateDoublesSketch sketch = DoublesSketch.builder().setK(2).build(); + sketch.update(doubleValue); + return sketch; + } + catch (NumberFormatException e) { + // Log.info("Expected Double. Got string with value " + + // objectString); + } + } + } + + if (object == null || object instanceof DoublesSketch || object instanceof Memory) { + return object; + } + return DoublesSketchOperations.deserialize(object); + } + }; + } + + @Override + public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder) + { + final GenericIndexed column = GenericIndexed.read(buffer, strategy); + builder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java new file mode 100644 index 000000000000..5cf2d8f0a61c --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java @@ -0,0 +1,78 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.yahoo.sketches.quantiles.UpdateDoublesSketch; + +import io.druid.query.aggregation.Aggregator; +import io.druid.segment.ColumnValueSelector; + +public class DoublesSketchDoubleAggregator implements Aggregator +{ + + private final ColumnValueSelector valueSelector; + private final int size; + + private UpdateDoublesSketch sketch; + + public DoublesSketchDoubleAggregator(final ColumnValueSelector valueSelector, final int size) + { + this.valueSelector = valueSelector; + this.size = size; + sketch = UpdateDoublesSketch.builder().setK(size).build(); + } + + @Override + public synchronized void aggregate() + { + sketch.update(valueSelector.getDouble()); + } + + @Override + public Object get() + { + return sketch; + } + + @Override + public float getFloat() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void reset() + { + sketch = UpdateDoublesSketch.builder().setK(size).build(); + } + + @Override + public void close() + { + sketch = null; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java new file mode 100644 index 000000000000..e10b6ee9bbd0 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java @@ -0,0 +1,138 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.nio.ByteBuffer; +import java.util.IdentityHashMap; + +import com.yahoo.memory.WritableMemory; +import com.yahoo.sketches.quantiles.UpdateDoublesSketch; + +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.ColumnValueSelector; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; + +public class DoublesSketchDoubleBufferAggregator implements BufferAggregator +{ + + private final ColumnValueSelector selector; + private final int size; + private final int maxIntermediateSize; + + private final IdentityHashMap memCache = new IdentityHashMap<>(); + private final IdentityHashMap> sketches = new IdentityHashMap<>(); + + public DoublesSketchDoubleBufferAggregator(final ColumnValueSelector valueSelector, final int size, + final int maxIntermediateSize) + { + this.selector = valueSelector; + this.size = size; + this.maxIntermediateSize = maxIntermediateSize; + } + + @Override + public void init(final ByteBuffer buffer, final int position) + { + final WritableMemory mem = getMemory(buffer); + final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); + final UpdateDoublesSketch sketch = UpdateDoublesSketch.builder().setK(size).build(region); + putSketch(buffer, position, sketch); + } + + @Override + public synchronized void aggregate(final ByteBuffer buffer, final int position) + { + final UpdateDoublesSketch sketch = sketches.get(buffer).get(position); + sketch.update(selector.getDouble()); + } + + @Override + public Object get(final ByteBuffer buffer, final int position) + { + return sketches.get(buffer).get(position); + } + + @Override + public float getFloat(final ByteBuffer buffer, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong(final ByteBuffer buffer, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() + { + sketches.clear(); + memCache.clear(); + } + + @Override + public void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) + { + UpdateDoublesSketch sketch = sketches.get(oldBuffer).get(oldPosition); + final WritableMemory oldRegion = getMemory(oldBuffer).writableRegion(oldPosition, maxIntermediateSize); + if (sketch.isSameResource(oldRegion)) { // sketch was not relocated on heap + final WritableMemory newRegion = getMemory(newBuffer).writableRegion(newPosition, maxIntermediateSize); + sketch = UpdateDoublesSketch.wrap(newRegion); + } + putSketch(newBuffer, newPosition, sketch); + + final Int2ObjectMap map = sketches.get(oldBuffer); + map.remove(oldPosition); + if (map.isEmpty()) { + sketches.remove(oldBuffer); + memCache.remove(oldBuffer); + } + } + + private WritableMemory getMemory(final ByteBuffer buffer) + { + WritableMemory mem = memCache.get(buffer); + if (mem == null) { + mem = WritableMemory.wrap(buffer); + memCache.put(buffer, mem); + } + return mem; + } + + private void putSketch(final ByteBuffer buffer, final int position, final UpdateDoublesSketch sketch) + { + Int2ObjectMap map = sketches.get(buffer); + if (map == null) { + map = new Int2ObjectOpenHashMap<>(); + sketches.put(buffer, map); + } + map.put(position, sketch); + } + + @Override + public void inspectRuntimeShape(final RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java new file mode 100644 index 000000000000..911b2ba62769 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java @@ -0,0 +1,160 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.cache.CacheKeyBuilder; + +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchHistogramPostAggregator implements PostAggregator +{ + + private static final byte CACHE_KEY = 112; + + private final String name; + private final PostAggregator field; + private final double[] splitPoints; + + @JsonCreator + public DoublesSketchHistogramPostAggregator( + @JsonProperty("name") final String name, + @JsonProperty("field") final PostAggregator field, + @JsonProperty("splitPoints") final double[] splitPoints) + { + this.name = Preconditions.checkNotNull(name, "name is null"); + this.field = Preconditions.checkNotNull(field, "field is null"); + this.splitPoints = Preconditions.checkNotNull(splitPoints, "array of split points is null"); + } + + @Override + public Object compute(final Map combinedAggregators) + { + final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); + final double[] histogram = sketch.getPMF(splitPoints); + for (int i = 0; i < histogram.length; i++) { + histogram[i] *= sketch.getN(); + } + return histogram; + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public PostAggregator getField() + { + return field; + } + + @JsonProperty + public double[] getSplitPoints() + { + return splitPoints; + } + + // comparing histograms doesn't make much sense, so this comparator pretends that everything is equal + @Override + public Comparator getComparator() + { + return new Comparator() + { + @Override + public int compare(double[] o1, double[] o2) + { + return 0; + } + }; + } + + @Override + public Set getDependentFields() + { + return field.getDependentFields(); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "name='" + name + '\'' + + ", field=" + field + + ", splitPoints=" + Arrays.toString(splitPoints) + + "}"; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DoublesSketchHistogramPostAggregator that = (DoublesSketchHistogramPostAggregator) o; + if (!name.equals(that.name)) { + return false; + } + if (!Arrays.equals(splitPoints, that.splitPoints)) { + return false; + } + return field.equals(that.field); + } + + @Override + public int hashCode() + { + int hashCode = name.hashCode() * 31 + field.hashCode(); + hashCode = hashCode * 31 + Arrays.hashCode(splitPoints); + return hashCode; + } + + @Override + public byte[] getCacheKey() + { + final CacheKeyBuilder builder = new CacheKeyBuilder(CACHE_KEY).appendCacheable(field); + for (final double value: splitPoints) { + builder.appendDouble(value); + } + return builder.build(); + } + + @Override + public PostAggregator decorate(Map map) + { + return this; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java new file mode 100644 index 000000000000..dbee8d91cc99 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java @@ -0,0 +1,40 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchJsonSerializer extends JsonSerializer +{ + + @Override + public void serialize(final DoublesSketch sketch, final JsonGenerator generator, final SerializerProvider provider) + throws IOException, JsonProcessingException + { + generator.writeBinary(sketch.toByteArray(true)); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java new file mode 100644 index 000000000000..c988bcefbe32 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java @@ -0,0 +1,65 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Arrays; +import java.util.List; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import com.yahoo.sketches.quantiles.DoublesSketch; + +import io.druid.initialization.DruidModule; +import io.druid.segment.serde.ComplexMetrics; + +public class DoublesSketchModule implements DruidModule +{ + + public static final String DOUBLES_SKETCH = "quantilesDoublesSketch"; + + public static final String DOUBLES_SKETCH_HISTOGRAM_POST_AGG = "quantilesDoublesSketchToHistogram"; + public static final String DOUBLES_SKETCH_QUANTILE_POST_AGG = "quantilesDoublesSketchToQuantile"; + public static final String DOUBLES_SKETCH_QUANTILES_POST_AGG = "quantilesDoublesSketchToQuantiles"; + public static final String DOUBLES_SKETCH_TO_STRING_POST_AGG = "quantilesDoublesSketchToString"; + + @Override + public void configure(final Binder binder) + { + if (ComplexMetrics.getSerdeForType(DOUBLES_SKETCH) == null) { + ComplexMetrics.registerSerde(DOUBLES_SKETCH, new DoublesSketchComplexMetricSerde()); + } + } + + @Override + public List getJacksonModules() + { + return Arrays. asList( + new SimpleModule("DoublesQuantilesSketchModule").registerSubtypes( + new NamedType(DoublesSketchAggregatorFactory.class, DOUBLES_SKETCH), + new NamedType(DoublesSketchHistogramPostAggregator.class, DOUBLES_SKETCH_HISTOGRAM_POST_AGG), + new NamedType(DoublesSketchQuantilePostAggregator.class, DOUBLES_SKETCH_QUANTILE_POST_AGG), + new NamedType(DoublesSketchQuantilesPostAggregator.class, DOUBLES_SKETCH_QUANTILES_POST_AGG), + new NamedType(DoublesSketchToStringPostAggregator.class, DOUBLES_SKETCH_TO_STRING_POST_AGG)) + .addSerializer(DoublesSketch.class, new DoublesSketchJsonSerializer())); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java new file mode 100644 index 000000000000..3f89c905803f --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java @@ -0,0 +1,64 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.nio.ByteBuffer; + +import com.yahoo.memory.Memory; +import com.yahoo.sketches.quantiles.DoublesSketch; + +import io.druid.segment.data.ObjectStrategy; + +public class DoublesSketchObjectStrategy implements ObjectStrategy +{ + + private static final byte[] EMPTY_BYTES = new byte[] {}; + + @Override + public int compare(final DoublesSketch s1, final DoublesSketch s2) + { + return DoublesSketchAggregatorFactory.COMPARATOR.compare(s1, s2); + } + + @Override + public DoublesSketch fromByteBuffer(final ByteBuffer buffer, final int numBytes) + { + if (numBytes == 0) { + return DoublesSketchOperations.EMPTY_SKETCH; + } + return DoublesSketch.wrap(Memory.wrap(buffer).region(buffer.position(), numBytes)); + } + + @Override + public Class getClazz() + { + return DoublesSketch.class; + } + + @Override + public byte[] toBytes(final DoublesSketch sketch) + { + if (sketch == null || sketch.isEmpty()) { + return EMPTY_BYTES; + } + return sketch.toByteArray(true); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java new file mode 100644 index 000000000000..706dd1d33622 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java @@ -0,0 +1,57 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import org.apache.commons.codec.binary.Base64; + +import com.google.common.base.Charsets; +import com.yahoo.memory.Memory; +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchOperations +{ + + public static final DoublesSketch EMPTY_SKETCH = DoublesSketch.builder().build(); + + public static DoublesSketch deserialize(final Object serializedSketch) + { + if (serializedSketch instanceof String) { + return deserializeFromBase64EncodedString((String) serializedSketch); + } else if (serializedSketch instanceof byte[]) { + return deserializeFromByteArray((byte[]) serializedSketch); + } else if (serializedSketch instanceof DoublesSketch) { + return (DoublesSketch) serializedSketch; + } + throw new IllegalStateException( + "Object is not of a type that can be deserialized to a quantiles DoublsSketch: " + + serializedSketch.getClass()); + } + + public static DoublesSketch deserializeFromBase64EncodedString(final String str) + { + return deserializeFromByteArray(Base64.decodeBase64(str.getBytes(Charsets.UTF_8))); + } + + public static DoublesSketch deserializeFromByteArray(final byte[] data) + { + return DoublesSketch.wrap(Memory.wrap(data)); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java new file mode 100644 index 000000000000..9ee94e28c1f1 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java @@ -0,0 +1,149 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.primitives.Doubles; + +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.cache.CacheKeyBuilder; + +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchQuantilePostAggregator implements PostAggregator +{ + + private static final byte CACHE_KEY = 113; + + private final String name; + private final PostAggregator field; + private final double fraction; + + @JsonCreator + public DoublesSketchQuantilePostAggregator( + @JsonProperty("name") final String name, + @JsonProperty("field") final PostAggregator field, + @JsonProperty("fraction") final double fraction) + { + this.name = Preconditions.checkNotNull(name, "name is null"); + this.field = Preconditions.checkNotNull(field, "field is null"); + this.fraction = fraction; + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public PostAggregator getField() + { + return field; + } + + @JsonProperty + public double getFraction() + { + return fraction; + } + + @Override + public Object compute(final Map combinedAggregators) + { + final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); + return sketch.getQuantile(fraction); + } + + @Override + public Comparator getComparator() + { + return new Comparator() + { + @Override + public int compare(Double a, Double b) + { + return Doubles.compare(a, b); + } + }; + } + + @Override + public Set getDependentFields() + { + return field.getDependentFields(); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "name='" + name + '\'' + + ", field=" + field + + ", fraction=" + fraction + + "}"; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DoublesSketchQuantilePostAggregator that = (DoublesSketchQuantilePostAggregator) o; + if (!name.equals(that.name)) { + return false; + } + if (fraction != that.fraction) { + return false; + } + return field.equals(that.field); + } + + @Override + public int hashCode() + { + return (name.hashCode() * 31 + field.hashCode()) * 31 + Double.hashCode(fraction); + } + + @Override + public byte[] getCacheKey() + { + return new CacheKeyBuilder(CACHE_KEY).appendCacheable(field).appendDouble(fraction).build(); + } + + @Override + public PostAggregator decorate(Map map) + { + return this; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java new file mode 100644 index 000000000000..35f872fe4ad3 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java @@ -0,0 +1,155 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.cache.CacheKeyBuilder; + +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchQuantilesPostAggregator implements PostAggregator +{ + + private static final byte CACHE_KEY = 114; + + private final String name; + private final PostAggregator field; + private final double[] fractions; + + @JsonCreator + public DoublesSketchQuantilesPostAggregator( + @JsonProperty("name") final String name, + @JsonProperty("field") final PostAggregator field, + @JsonProperty("fractions") final double[] fractions) + { + this.name = Preconditions.checkNotNull(name, "name is null"); + this.field = Preconditions.checkNotNull(field, "field is null"); + this.fractions = Preconditions.checkNotNull(fractions, "array of fractions is null"); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public PostAggregator getField() + { + return field; + } + + @JsonProperty + public double[] getFractions() + { + return fractions; + } + + @Override + public Object compute(final Map combinedAggregators) + { + final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); + return sketch.getQuantiles(fractions); + } + + // comparing arrays of quantiles doesn't make much sense, so this comparator + // pretends everything is equal + @Override + public Comparator getComparator() + { + return new Comparator() + { + @Override + public int compare(double[] o1, double[] o2) + { + return 0; + } + }; + } + + @Override + public Set getDependentFields() + { + return field.getDependentFields(); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "name='" + name + '\'' + + ", field=" + field + + ", fractions=" + Arrays.toString(fractions) + + "}"; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DoublesSketchQuantilesPostAggregator that = (DoublesSketchQuantilesPostAggregator) o; + if (!name.equals(that.name)) { + return false; + } + if (!Arrays.equals(fractions, that.fractions)) { + return false; + } + return field.equals(that.field); + } + + @Override + public int hashCode() + { + return (name.hashCode() * 31 + field.hashCode()) * 31 + Arrays.hashCode(fractions); + } + + @Override + public byte[] getCacheKey() + { + final CacheKeyBuilder builder = new CacheKeyBuilder(CACHE_KEY).appendCacheable(field); + for (final double value: fractions) { + builder.appendDouble(value); + } + return builder.build(); + } + + @Override + public PostAggregator decorate(Map map) + { + return this; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java new file mode 100644 index 000000000000..6c56dfdd2d7c --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java @@ -0,0 +1,138 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.cache.CacheKeyBuilder; + +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchToStringPostAggregator implements PostAggregator +{ + + private static final byte CACHE_KEY = 115; + + private final String name; + private final PostAggregator field; + + @JsonCreator + public DoublesSketchToStringPostAggregator( + @JsonProperty("name") final String name, + @JsonProperty("field") final PostAggregator field) + { + this.name = Preconditions.checkNotNull(name, "name is null"); + this.field = Preconditions.checkNotNull(field, "field is null"); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public PostAggregator getField() + { + return field; + } + + @Override + public Object compute(final Map combinedAggregators) + { + final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); + return sketch.toString(); + } + + // comparing sketch summaries doesn't make much sense, so this comparator + // pretends everything is equal + @Override + public Comparator getComparator() + { + return new Comparator() + { + @Override + public int compare(String o1, String o2) + { + return 0; + } + }; + } + + @Override + public Set getDependentFields() + { + return field.getDependentFields(); + } + + @Override + public String toString() + { + return this.getClass().getSimpleName() + "{" + + "name='" + name + '\'' + + ", field=" + field + + "}"; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DoublesSketchToStringPostAggregator that = (DoublesSketchToStringPostAggregator) o; + if (!name.equals(that.name)) { + return false; + } + return field.equals(that.field); + } + + @Override + public int hashCode() + { + return (name.hashCode() * 31 + field.hashCode()) * 31; + } + + @Override + public byte[] getCacheKey() + { + final CacheKeyBuilder builder = new CacheKeyBuilder(CACHE_KEY).appendCacheable(field); + return builder.build(); + } + + @Override + public PostAggregator decorate(Map map) + { + return this; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchAggregator.java new file mode 100644 index 000000000000..efb024d61036 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchAggregator.java @@ -0,0 +1,60 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import io.druid.query.aggregation.Aggregator; + +public class EmptyDoublesSketchAggregator implements Aggregator +{ + + @Override + public Object get() + { + return DoublesSketchOperations.EMPTY_SKETCH; + } + + @Override + public void aggregate() + { + } + + @Override + public void reset() + { + } + + @Override + public void close() + { + } + + @Override + public float getFloat() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong() + { + throw new UnsupportedOperationException("Not implemented"); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchBufferAggregator.java new file mode 100644 index 000000000000..4c34f787ad15 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchBufferAggregator.java @@ -0,0 +1,68 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.nio.ByteBuffer; + +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; + +public class EmptyDoublesSketchBufferAggregator implements BufferAggregator +{ + + @Override + public void init(final ByteBuffer buf, final int position) + { + } + + @Override + public void aggregate(final ByteBuffer buf, final int position) + { + } + + @Override + public Object get(final ByteBuffer buf, final int position) + { + return DoublesSketchOperations.EMPTY_SKETCH; + } + + @Override + public float getFloat(final ByteBuffer buf, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong(final ByteBuffer buf, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() + { + } + + @Override + public void inspectRuntimeShape(final RuntimeShapeInspector inspector) + { + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java index 4b613d848a6c..6fa21fec6bbe 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java @@ -21,13 +21,14 @@ import com.yahoo.memory.Memory; import com.yahoo.memory.WritableMemory; +import com.yahoo.sketches.Family; import com.yahoo.sketches.theta.CompactSketch; import com.yahoo.sketches.theta.Sketch; import com.yahoo.sketches.theta.Union; /** */ -public class SynchronizedUnion implements Union +public class SynchronizedUnion extends Union { private final Union delegate; @@ -119,4 +120,11 @@ public synchronized boolean isSameResource(Memory mem) { return delegate.isSameResource(mem); } + + @Override + public synchronized Family getFamily() + { + return delegate.getFamily(); + } + } diff --git a/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule index efd10f3a7e78..dfc5b69e00fb 100644 --- a/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule +++ b/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule @@ -1,2 +1,3 @@ io.druid.query.aggregation.datasketches.theta.SketchModule io.druid.query.aggregation.datasketches.theta.oldapi.OldApiSketchModule +io.druid.query.aggregation.datasketches.quantiles.DoublesSketchModule diff --git a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java new file mode 100644 index 000000000000..45ceb6e258ac --- /dev/null +++ b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java @@ -0,0 +1,423 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import io.druid.data.input.Row; +import io.druid.initialization.DruidModule; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.java.util.common.granularity.Granularities; +import io.druid.java.util.common.guava.Sequence; +import io.druid.java.util.common.guava.Sequences; +import io.druid.query.aggregation.AggregationTestHelper; +import io.druid.query.groupby.GroupByQueryConfig; +import io.druid.query.groupby.GroupByQueryRunnerTest; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@RunWith(Parameterized.class) +public class DoublesSketchAggregatorTest +{ + + private final AggregationTestHelper helper; + private final AggregationTestHelper timeSeriesHelper; + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + public DoublesSketchAggregatorTest(final GroupByQueryConfig config) + { + DruidModule module = new DoublesSketchModule(); + module.configure(null); + helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( + module.getJacksonModules(), config, tempFolder); + timeSeriesHelper = AggregationTestHelper.createTimeseriesQueryAggregationTestHelper( + module.getJacksonModules(), + tempFolder); + } + + @Parameterized.Parameters(name = "{0}") + public static Collection constructorFeeder() + { + final List constructors = Lists.newArrayList(); + for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { + constructors.add(new Object[] {config}); + } + return constructors; + } + + // this is to test Json properties and equals + @Test + public void serializeDeserializeFactoryWithFieldName() throws Exception + { + ObjectMapper objectMapper = new DefaultObjectMapper(); + DoublesSketchAggregatorFactory factory = new DoublesSketchAggregatorFactory("name", "filedName", 128); + + DoublesSketchAggregatorFactory other = objectMapper.readValue( + objectMapper.writeValueAsString(factory), + DoublesSketchAggregatorFactory.class); + + Assert.assertEquals(factory, other); + } + + @Test + public void ingestingSketches() throws Exception + { + Sequence seq = helper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_sketch_data.tsv").getFile()), + String.join("\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"product\"],", + " \"dimensionExclusions\": [],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"product\", \"sketch\"]", + " }", + "}"), + String.join("\n", + "[", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 128},", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"non_existent_sketch\", \"fieldName\": \"non_existent_sketch\", \"k\": 128}", + "]"), + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join("\n", + "{", + " \"queryType\": \"groupBy\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"dimensions\": [],", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 128},", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"non_existent_sketch\", \"fieldName\": \"non_existent_sketch\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + Row row = results.get(0); + + Object nonExistentSketchObject = row.getRaw("non_existent_sketch"); + Assert.assertTrue(nonExistentSketchObject instanceof Long); + long nonExistentSketchValue = (long) nonExistentSketchObject; + Assert.assertEquals(0, nonExistentSketchValue); + + Object sketchObject = row.getRaw("sketch"); + Assert.assertTrue(sketchObject instanceof Long); + long sketchValue = (long) sketchObject; + Assert.assertEquals(400, sketchValue); + + // post agg + Object quantilesObject = row.getRaw("quantiles"); + Assert.assertTrue(quantilesObject instanceof double[]); + double[] quantiles = (double[]) quantilesObject; + Assert.assertEquals(0, quantiles[0], 0.05); // min value + Assert.assertEquals(0.5, quantiles[1], 0.05); // median value + Assert.assertEquals(1, quantiles[2], 0.05); // max value + + // post agg + Object histogramObject = row.getRaw("histogram"); + Assert.assertTrue(histogramObject instanceof double[]); + double[] histogram = (double[]) histogramObject; + for (int i = 0; i < histogram.length; i++) { + Assert.assertEquals(100, histogram[i], 100 * 0.2); // 400 items uniformly + // distributed into 4 + // bins + } + } + + @Test + public void buildingSketchesAtIngestionTime() throws Exception + { + Sequence seq = helper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), + String.join("\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"product\"],", + " \"dimensionExclusions\": [ \"sequenceNumber\"],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", + " }", + "}"), + "[{\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}]", + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join("\n", + "{", + " \"queryType\": \"groupBy\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"dimensions\": [],", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 128},", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"non_existent_sketch\", \"fieldName\": \"non_existent_sketch\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + Row row = results.get(0); + + Object sketchObject = row.getRaw("sketch"); + Assert.assertTrue(sketchObject instanceof Long); + long sketchValue = (long) sketchObject; + Assert.assertEquals(400, sketchValue); + + // post agg + Object quantilesObject = row.getRaw("quantiles"); + Assert.assertTrue(quantilesObject instanceof double[]); + double[] quantiles = (double[]) quantilesObject; + Assert.assertEquals(0, quantiles[0], 0.05); // min value + Assert.assertEquals(0.5, quantiles[1], 0.05); // median value + Assert.assertEquals(1, quantiles[2], 0.05); // max value + + // post agg + Object histogramObject = row.getRaw("histogram"); + Assert.assertTrue(histogramObject instanceof double[]); + double[] histogram = (double[]) histogramObject; + Assert.assertEquals(4, histogram.length); + for (int i = 0; i < histogram.length; i++) { + Assert.assertEquals(100, histogram[i], 100 * 0.2); // 400 items uniformly + // distributed into 4 + // bins + } + } + + @Test + public void buildingSketchesAtQueryTime() throws Exception + { + Sequence seq = helper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), + String.join("\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"sequenceNumber\", \"product\"],", + " \"dimensionExclusions\": [],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", + " }", + "}"), + "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join("\n", + "{", + " \"queryType\": \"groupBy\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"dimensions\": [],", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantile\", \"name\": \"quantile\", \"fraction\": 0.5, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + Row row = results.get(0); + + Object sketchObject = row.getRaw("sketch"); + Assert.assertTrue(sketchObject instanceof Long); + long sketchValue = (long) sketchObject; + Assert.assertEquals(400, sketchValue); + + // post agg + Object quantileObject = row.getRaw("quantile"); + Assert.assertTrue(quantileObject instanceof Double); + Assert.assertEquals(0.5, (double) quantileObject, 0.05); // median value + + // post agg + Object quantilesObject = row.getRaw("quantiles"); + Assert.assertTrue(quantilesObject instanceof double[]); + double[] quantiles = (double[]) quantilesObject; + Assert.assertEquals(0, quantiles[0], 0.05); // min value + Assert.assertEquals(0.5, quantiles[1], 0.05); // median value + Assert.assertEquals(1, quantiles[2], 0.05); // max value + + // post agg + Object histogramObject = row.getRaw("histogram"); + Assert.assertTrue(histogramObject instanceof double[]); + double[] histogram = (double[]) histogramObject; + for (int i = 0; i < histogram.length; i++) { + Assert.assertEquals(100, histogram[i], 100 * 0.2); // 400 items uniformly + // distributed into 4 + // bins + } + } + + @Test + public void QueryingDataWithFieldNameValueAsFloatInsteadOfSketch() throws Exception + { + Sequence seq = helper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), + String.join( + "\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"sequenceNumber\", \"product\"],", + " \"dimensionExclusions\": [],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", + " }", + "}"), + "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join( + "\n", + "{", + " \"queryType\": \"groupBy\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"dimensions\": [],", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantile\", \"name\": \"quantile\", \"fraction\": 0.5, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + Row row = results.get(0); + + Object sketchObject = row.getRaw("sketch"); + Assert.assertTrue(sketchObject instanceof Long); + long sketchValue = (long) sketchObject; + Assert.assertEquals(400, sketchValue); + + // post agg + Object quantileObject = row.getRaw("quantile"); + Assert.assertTrue(quantileObject instanceof Double); + Assert.assertEquals(0.5, (double) quantileObject, 0.05); // median value + + // post agg + Object quantilesObject = row.getRaw("quantiles"); + Assert.assertTrue(quantilesObject instanceof double[]); + double[] quantiles = (double[]) quantilesObject; + Assert.assertEquals(0, quantiles[0], 0.05); // min value + Assert.assertEquals(0.5, quantiles[1], 0.05); // median value + Assert.assertEquals(1, quantiles[2], 0.05); // max value + + // post agg + Object histogramObject = row.getRaw("histogram"); + Assert.assertTrue(histogramObject instanceof double[]); + double[] histogram = (double[]) histogramObject; + for (int i = 0; i < histogram.length; i++) { + Assert.assertEquals(100, histogram[i], 100 * 0.2); // 400 items uniformly + // distributed into 4 + // bins + } + } + + @Test + public void TimeSeriesQueryInputAsFloat() throws Exception + { + Sequence seq = timeSeriesHelper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), + String.join( + "\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"sequenceNumber\", \"product\"],", + " \"dimensionExclusions\": [],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", + " }", + "}"), + "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join( + "\n", + "{", + " \"queryType\": \"timeseries\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantile\", \"name\": \"quantile1\", \"fraction\": 0.5, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles1\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram1\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + } +} diff --git a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java new file mode 100644 index 000000000000..eed492c9a9f9 --- /dev/null +++ b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java @@ -0,0 +1,71 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Random; + +import org.apache.commons.codec.binary.Base64; + +import com.yahoo.sketches.quantiles.UpdateDoublesSketch; + +public class GenerateTestData +{ + + public static void main(String[] args) throws Exception + { + Path buildPath = FileSystems.getDefault().getPath("doubles_build_data.tsv"); + Path sketchPath = FileSystems.getDefault().getPath("doubles_sketch_data.tsv"); + BufferedWriter buildData = Files.newBufferedWriter(buildPath, StandardCharsets.UTF_8); + BufferedWriter sketchData = Files.newBufferedWriter(sketchPath, StandardCharsets.UTF_8); + Random rand = new Random(); + int sequenceNumber = 0; + for (int i = 0; i < 20; i++) { + int product = rand.nextInt(10); + UpdateDoublesSketch sketch = UpdateDoublesSketch.builder().build(); + for (int j = 0; j < 20; j++) { + double value = rand.nextDouble(); + buildData.write("2016010101"); + buildData.write('\t'); + buildData.write(Integer.toString(sequenceNumber)); // dimension with unique numbers for ingesting raw data + buildData.write('\t'); + buildData.write(Integer.toString(product)); // product dimension + buildData.write('\t'); + buildData.write(Double.toString(value)); + buildData.newLine(); + sketch.update(value); + sequenceNumber++; + } + sketchData.write("2016010101"); + sketchData.write('\t'); + sketchData.write(Integer.toString(product)); // product dimension + sketchData.write('\t'); + sketchData.write(Base64.encodeBase64String(sketch.toByteArray(true))); + sketchData.newLine(); + } + buildData.close(); + sketchData.close(); + } + +} diff --git a/extensions-core/datasketches/src/test/resources/quantiles/doubles_build_data.tsv b/extensions-core/datasketches/src/test/resources/quantiles/doubles_build_data.tsv new file mode 100644 index 000000000000..bb59faf3da8b --- /dev/null +++ b/extensions-core/datasketches/src/test/resources/quantiles/doubles_build_data.tsv @@ -0,0 +1,400 @@ +2016010101 0 0 0.6529403005319299 +2016010101 1 0 0.9270214958987323 +2016010101 2 0 0.6383273609981486 +2016010101 3 0 0.8088289215633632 +2016010101 4 0 0.8163864917598281 +2016010101 5 0 0.38484848588530784 +2016010101 6 0 0.7690020468986823 +2016010101 7 0 0.6212078833139824 +2016010101 8 0 0.4915825094949512 +2016010101 9 0 0.688004059332008 +2016010101 10 0 0.2536908275250508 +2016010101 11 0 0.6618435914290263 +2016010101 12 0 0.7892773595797635 +2016010101 13 0 0.08857624134076048 +2016010101 14 0 0.11992633801904151 +2016010101 15 0 0.4959192800105586 +2016010101 16 0 0.5564893557708243 +2016010101 17 0 0.7755547456799993 +2016010101 18 0 0.06420706406984311 +2016010101 19 0 0.23085639094262378 +2016010101 20 7 0.012013916725163498 +2016010101 21 7 0.34077219818209503 +2016010101 22 7 0.8445966884204918 +2016010101 23 7 0.6466142718287953 +2016010101 24 7 0.43959032391415487 +2016010101 25 7 0.7768829233737787 +2016010101 26 7 0.5899544206136442 +2016010101 27 7 0.017782361911801825 +2016010101 28 7 0.5431916165782864 +2016010101 29 7 0.8218253174439416 +2016010101 30 7 0.6372788284951859 +2016010101 31 7 0.41403671834680933 +2016010101 32 7 0.042508330730374855 +2016010101 33 7 0.7416290691530969 +2016010101 34 7 0.6990557213726277 +2016010101 35 7 0.6302154208823348 +2016010101 36 7 0.021053567154993402 +2016010101 37 7 0.770280353784988 +2016010101 38 7 0.08205576978448703 +2016010101 39 7 0.2049660800682488 +2016010101 40 5 0.08129304678049831 +2016010101 41 5 0.17754747271638005 +2016010101 42 5 0.8441702357096768 +2016010101 43 5 0.9060464737257796 +2016010101 44 5 0.5970595512785409 +2016010101 45 5 0.843859346312315 +2016010101 46 5 0.1649847892987305 +2016010101 47 5 0.5279903496999094 +2016010101 48 5 0.08758749830556767 +2016010101 49 5 0.6088480522002063 +2016010101 50 5 0.31079133043670004 +2016010101 51 5 0.43062105356651226 +2016010101 52 5 0.8542989852099488 +2016010101 53 5 0.42443162807834045 +2016010101 54 5 0.5020327054358468 +2016010101 55 5 0.36453920012074237 +2016010101 56 5 0.9884597580348689 +2016010101 57 5 0.3770559586575706 +2016010101 58 5 0.5989237303385875 +2016010101 59 5 0.9926342802399872 +2016010101 60 4 0.7813961047849703 +2016010101 61 4 0.062171533805525425 +2016010101 62 4 0.5284977503473608 +2016010101 63 4 0.5924687065581794 +2016010101 64 4 0.06305234223879275 +2016010101 65 4 0.4959562731747129 +2016010101 66 4 0.6336733165353365 +2016010101 67 4 0.48860263540869875 +2016010101 68 4 0.9387610528974851 +2016010101 69 4 0.3391271652731308 +2016010101 70 4 0.5962837638971421 +2016010101 71 4 0.9190447294921896 +2016010101 72 4 0.33082943548872534 +2016010101 73 4 0.6236359023672029 +2016010101 74 4 0.27134427542016615 +2016010101 75 4 0.11665530238761901 +2016010101 76 4 0.10469260335277608 +2016010101 77 4 0.6824658847771211 +2016010101 78 4 0.6131047630496756 +2016010101 79 4 0.9838171536972515 +2016010101 80 4 0.7484669110852756 +2016010101 81 4 0.797620888697219 +2016010101 82 4 0.7166673353657907 +2016010101 83 4 0.46968710353176557 +2016010101 84 4 0.3998491199643106 +2016010101 85 4 0.6314883585976869 +2016010101 86 4 0.8305617875577815 +2016010101 87 4 0.6867651870284084 +2016010101 88 4 0.9961677044887979 +2016010101 89 4 0.19745766301180412 +2016010101 90 4 0.2737652043079263 +2016010101 91 4 0.2954503444695358 +2016010101 92 4 0.6191902196833489 +2016010101 93 4 0.6828058006233482 +2016010101 94 4 0.7967115641510757 +2016010101 95 4 0.5485460823820962 +2016010101 96 4 0.4278132830938558 +2016010101 97 4 0.32194908458166194 +2016010101 98 4 0.07094920295725238 +2016010101 99 4 0.4351839393889565 +2016010101 100 1 0.6160833396611648 +2016010101 101 1 0.4652667787803648 +2016010101 102 1 0.5026953463132913 +2016010101 103 1 0.4103237191034753 +2016010101 104 1 0.3298554666697301 +2016010101 105 1 0.16907537273919138 +2016010101 106 1 0.6945260598989513 +2016010101 107 1 0.917138530496438 +2016010101 108 1 0.8810129148605083 +2016010101 109 1 0.11845626048380542 +2016010101 110 1 0.8848971155827816 +2016010101 111 1 0.9969103769603667 +2016010101 112 1 0.06274198529295416 +2016010101 113 1 0.2923616769686519 +2016010101 114 1 0.12621083638328634 +2016010101 115 1 0.9655188575577313 +2016010101 116 1 0.6074995164352884 +2016010101 117 1 0.5501887988201414 +2016010101 118 1 0.9406914128003497 +2016010101 119 1 0.03264873659277656 +2016010101 120 6 0.004852543443656487 +2016010101 121 6 0.11161194329252788 +2016010101 122 6 0.9403527002796559 +2016010101 123 6 0.8951866979503953 +2016010101 124 6 0.07629846897033454 +2016010101 125 6 0.9898485014275873 +2016010101 126 6 0.42827377712188075 +2016010101 127 6 0.4274796777951825 +2016010101 128 6 0.5569522946332676 +2016010101 129 6 0.028195121559112635 +2016010101 130 6 0.8599127909482382 +2016010101 131 6 0.3516112293128607 +2016010101 132 6 0.3888868189342449 +2016010101 133 6 0.644589126160206 +2016010101 134 6 0.7398741071492928 +2016010101 135 6 0.1998479248216123 +2016010101 136 6 0.8803215884594476 +2016010101 137 6 0.7079531966558515 +2016010101 138 6 0.7904290564015343 +2016010101 139 6 0.475671788742007 +2016010101 140 3 0.034708334899357096 +2016010101 141 3 0.4134637419532796 +2016010101 142 3 0.9757934592902832 +2016010101 143 3 0.37422347371609666 +2016010101 144 3 0.5904996168737154 +2016010101 145 3 0.5883259679727514 +2016010101 146 3 0.3380286015499171 +2016010101 147 3 0.42174393035143043 +2016010101 148 3 0.4764900074141757 +2016010101 149 3 0.01864239537224921 +2016010101 150 3 0.9124007087743986 +2016010101 151 3 0.8951275235699193 +2016010101 152 3 0.7037272142266654 +2016010101 153 3 0.5685506209266902 +2016010101 154 3 0.4104883958833594 +2016010101 155 3 0.7794005551450208 +2016010101 156 3 0.2879354697088996 +2016010101 157 3 0.5243215707259823 +2016010101 158 3 0.22238840286136063 +2016010101 159 3 0.11336472553284738 +2016010101 160 4 0.9800770037725316 +2016010101 161 4 0.7628237317889158 +2016010101 162 4 0.5355335935170453 +2016010101 163 4 0.9676939330565402 +2016010101 164 4 0.657825753108034 +2016010101 165 4 0.9175328548944673 +2016010101 166 4 0.6834666043257283 +2016010101 167 4 0.08580759367942314 +2016010101 168 4 0.3134740602060899 +2016010101 169 4 0.3218818254752742 +2016010101 170 4 0.6119297354994999 +2016010101 171 4 0.07086832750773142 +2016010101 172 4 0.2700864307032772 +2016010101 173 4 0.7497315076673637 +2016010101 174 4 0.4959921300968493 +2016010101 175 4 0.09294825796093753 +2016010101 176 4 0.4954515904444161 +2016010101 177 4 0.8820366880191506 +2016010101 178 4 0.17978298283728522 +2016010101 179 4 0.05259679741524781 +2016010101 180 5 0.4711892966981096 +2016010101 181 5 0.5965662941715105 +2016010101 182 5 0.4775201668966973 +2016010101 183 5 0.05084576687030873 +2016010101 184 5 0.16680660677593928 +2016010101 185 5 0.9342287333653685 +2016010101 186 5 0.8153161893769392 +2016010101 187 5 0.9362517669519288 +2016010101 188 5 0.10865218471840699 +2016010101 189 5 0.44665378915111065 +2016010101 190 5 0.8804454791937898 +2016010101 191 5 0.20666928346935398 +2016010101 192 5 0.7052479677101612 +2016010101 193 5 0.5006205470200923 +2016010101 194 5 0.23220501028575968 +2016010101 195 5 0.11776507130391467 +2016010101 196 5 0.592011744069295 +2016010101 197 5 0.7089191450076786 +2016010101 198 5 0.7269340552231702 +2016010101 199 5 0.7049554871226075 +2016010101 200 1 0.44078367400761076 +2016010101 201 1 0.7715264806037321 +2016010101 202 1 0.10151701902103971 +2016010101 203 1 0.661891806135609 +2016010101 204 1 0.23095745116331567 +2016010101 205 1 0.46625278601359255 +2016010101 206 1 0.5912486124707177 +2016010101 207 1 0.963946871892115 +2016010101 208 1 0.8172596270687692 +2016010101 209 1 0.05745699928199144 +2016010101 210 1 0.40612684342877337 +2016010101 211 1 0.6330844777969608 +2016010101 212 1 0.3148973406065705 +2016010101 213 1 0.23230462811318175 +2016010101 214 1 0.9960772952945196 +2016010101 215 1 0.4581376339786414 +2016010101 216 1 0.7181494575770677 +2016010101 217 1 0.04277917580280799 +2016010101 218 1 0.11137419446625674 +2016010101 219 1 0.014716278313423037 +2016010101 220 2 0.8988603727313186 +2016010101 221 2 0.8192124226306603 +2016010101 222 2 0.9304683598956597 +2016010101 223 2 0.4375546733938238 +2016010101 224 2 0.7676359685332207 +2016010101 225 2 0.30977859822027964 +2016010101 226 2 0.008595955287459267 +2016010101 227 2 0.6790605343724216 +2016010101 228 2 0.36949588946147993 +2016010101 229 2 0.3826798435706562 +2016010101 230 2 0.13836513167087128 +2016010101 231 2 0.4451570472364902 +2016010101 232 2 0.8944067771338549 +2016010101 233 2 0.6068095655362902 +2016010101 234 2 0.7084870042917992 +2016010101 235 2 0.5867363290655241 +2016010101 236 2 0.6903863088381504 +2016010101 237 2 0.30984947936089124 +2016010101 238 2 0.31561088279452665 +2016010101 239 2 0.006286479849849758 +2016010101 240 5 0.34397466439693725 +2016010101 241 5 0.052476003295899964 +2016010101 242 5 0.726106045184451 +2016010101 243 5 0.01559115401009159 +2016010101 244 5 0.9219270739836661 +2016010101 245 5 0.5147917330760431 +2016010101 246 5 0.41919804470784205 +2016010101 247 5 0.4145101775865617 +2016010101 248 5 0.34153038022995796 +2016010101 249 5 0.9503817180587767 +2016010101 250 5 0.6958354849389804 +2016010101 251 5 0.46000811480536297 +2016010101 252 5 0.18379911670616378 +2016010101 253 5 0.20973108758556713 +2016010101 254 5 0.5979201603287885 +2016010101 255 5 0.5552419362393491 +2016010101 256 5 0.10996555307297629 +2016010101 257 5 0.3591453585622102 +2016010101 258 5 0.06098055111386691 +2016010101 259 5 0.5227270267924988 +2016010101 260 0 0.8492702312836989 +2016010101 261 0 0.5941242001151825 +2016010101 262 0 0.6840733026822607 +2016010101 263 0 0.8109777000249937 +2016010101 264 0 0.8599286045013937 +2016010101 265 0 0.7828806670746145 +2016010101 266 0 0.8102260971867188 +2016010101 267 0 0.38306094770114385 +2016010101 268 0 0.7093609268723879 +2016010101 269 0 0.4806583187577358 +2016010101 270 0 0.5766489331365172 +2016010101 271 0 0.7565067278238041 +2016010101 272 0 0.8262768908267573 +2016010101 273 0 0.7951015619138146 +2016010101 274 0 0.1938448910588796 +2016010101 275 0 0.8884608583839426 +2016010101 276 0 0.7046203516594505 +2016010101 277 0 0.5951074760704175 +2016010101 278 0 0.38207409719784036 +2016010101 279 0 0.2445271560830221 +2016010101 280 7 0.6032919624054952 +2016010101 281 7 0.1473220747987144 +2016010101 282 7 0.38396643099307604 +2016010101 283 7 0.4431561135554619 +2016010101 284 7 0.896578318093225 +2016010101 285 7 0.6729206122043515 +2016010101 286 7 0.8498821349478478 +2016010101 287 7 0.48231924024179784 +2016010101 288 7 0.005379480238994816 +2016010101 289 7 0.8017936717647264 +2016010101 290 7 0.08193232952990348 +2016010101 291 7 0.3422943366454193 +2016010101 292 7 0.6081556855207957 +2016010101 293 7 0.641193222941943 +2016010101 294 7 0.3716858024654186 +2016010101 295 7 0.0011169303830090849 +2016010101 296 7 0.4698784438339285 +2016010101 297 7 0.958198841287214 +2016010101 298 7 0.730945048929339 +2016010101 299 7 0.1858601884405512 +2016010101 300 5 0.1020825694779407 +2016010101 301 5 0.5742385074938443 +2016010101 302 5 0.9846817584978909 +2016010101 303 5 0.3858694391491331 +2016010101 304 5 0.9822246873202894 +2016010101 305 5 0.39822015482143314 +2016010101 306 5 0.6575924137957005 +2016010101 307 5 0.02359557062746842 +2016010101 308 5 0.42059510563039115 +2016010101 309 5 0.5970764856116284 +2016010101 310 5 0.2817399870096221 +2016010101 311 5 0.5334091165258412 +2016010101 312 5 0.31199853410796585 +2016010101 313 5 0.3156991306990594 +2016010101 314 5 0.9560285139855889 +2016010101 315 5 0.7846951771498516 +2016010101 316 5 0.009731486767097897 +2016010101 317 5 0.22625857375026215 +2016010101 318 5 0.8580955944724618 +2016010101 319 5 0.9622008926137687 +2016010101 320 5 0.023872302930851297 +2016010101 321 5 0.3580981601151092 +2016010101 322 5 0.9120442264954038 +2016010101 323 5 0.5968491989965334 +2016010101 324 5 0.5028516120506729 +2016010101 325 5 0.30590552314314 +2016010101 326 5 0.5566430714368423 +2016010101 327 5 0.6441099124064397 +2016010101 328 5 0.8765287851559298 +2016010101 329 5 0.38405928947408385 +2016010101 330 5 0.29654203975364 +2016010101 331 5 0.3606921959261904 +2016010101 332 5 0.9617038824842609 +2016010101 333 5 0.3103700669261584 +2016010101 334 5 0.4935170174690311 +2016010101 335 5 0.34757561267296444 +2016010101 336 5 0.1236918485545484 +2016010101 337 5 0.24925258973306597 +2016010101 338 5 0.4104821367672965 +2016010101 339 5 0.3621850216936935 +2016010101 340 6 0.3816099229918041 +2016010101 341 6 0.9496667754823915 +2016010101 342 6 0.5594605720642025 +2016010101 343 6 0.8537860901562698 +2016010101 344 6 0.74787202967909 +2016010101 345 6 0.29699361421249604 +2016010101 346 6 0.035943527086235605 +2016010101 347 6 0.20106098029261277 +2016010101 348 6 0.6589994525818863 +2016010101 349 6 0.3851541727199762 +2016010101 350 6 0.12262059605539744 +2016010101 351 6 0.33383436408012057 +2016010101 352 6 0.5087733967157267 +2016010101 353 6 0.34978350071897446 +2016010101 354 6 0.9171509423859847 +2016010101 355 6 0.6395164525815664 +2016010101 356 6 0.659637993918835 +2016010101 357 6 0.5689746534857604 +2016010101 358 6 0.03266513163571427 +2016010101 359 6 0.5863675010868861 +2016010101 360 9 0.8665167898047901 +2016010101 361 9 0.7933960420424948 +2016010101 362 9 0.8409667771425247 +2016010101 363 9 0.9544310598825743 +2016010101 364 9 0.36206869840549716 +2016010101 365 9 0.253957983880155 +2016010101 366 9 0.08496022679431525 +2016010101 367 9 0.5483782518766319 +2016010101 368 9 0.41440902281408365 +2016010101 369 9 0.2947889064970717 +2016010101 370 9 0.659477180019486 +2016010101 371 9 0.9016744422830162 +2016010101 372 9 0.4692828259677926 +2016010101 373 9 0.4221974527778145 +2016010101 374 9 0.26318360778150285 +2016010101 375 9 0.10064081807071767 +2016010101 376 9 0.7781802619858804 +2016010101 377 9 0.529215767115243 +2016010101 378 9 0.21094147073619007 +2016010101 379 9 0.18894985078463877 +2016010101 380 5 0.20683422198832369 +2016010101 381 5 0.9506923735546904 +2016010101 382 5 0.25734447316063913 +2016010101 383 5 0.6439025323539892 +2016010101 384 5 0.9099080819805052 +2016010101 385 5 0.9331714165375404 +2016010101 386 5 0.24979840404324272 +2016010101 387 5 0.40270120064812764 +2016010101 388 5 0.35895113537427137 +2016010101 389 5 0.44814114645480074 +2016010101 390 5 0.437368419580639 +2016010101 391 5 0.2777496228001308 +2016010101 392 5 0.09350862521048608 +2016010101 393 5 0.10366624548706516 +2016010101 394 5 0.8715309310993357 +2016010101 395 5 0.8953111125914557 +2016010101 396 5 0.9410866942183567 +2016010101 397 5 0.16367286942347592 +2016010101 398 5 0.6995415361957786 +2016010101 399 5 0.7170527361072194 diff --git a/extensions-core/datasketches/src/test/resources/quantiles/doubles_sketch_data.tsv b/extensions-core/datasketches/src/test/resources/quantiles/doubles_sketch_data.tsv new file mode 100644 index 000000000000..18872e35d7b6 --- /dev/null +++ b/extensions-core/datasketches/src/test/resources/quantiles/doubles_sketch_data.tsv @@ -0,0 +1,20 @@ +2016010101 0 AgMIEIAAAAAUAAAAAAAAALAmWsjfb7A/pV7y+yiq7T+wJlrI32+wPwDcwrvurLY/+Ie4E36zvj+Q6JXEs4zNPy764HN4PNA/kLgyi1uh2D9OQGR8FnbfP1J6Rjgkvd8/F3Pzw8LO4T/jOtta7+DjPwwhdIAtbeQ/Ns2gDuPk5D/DJ4ic0i3lPzgUyxYhBOY/rZQ/Lqqb6D+RgJ4vWNHoP0DJ25fCQek/FY7FMO3h6T/5JWCQ1h/qP6Ve8vsoqu0/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 7 AgMIEIAAAAAUAAAAAAAAAECZc5vAmog/5G1iou8G6z9AmXObwJqIP4BjbRuKNZI/AFCU+RCPlT9AypbkpsOlP3AnrF+bAbU/ZFpXGVQ8yj8QAKUxNs/VPyYiKd2Tf9o/nls2dD8i3D+YSJVi02HhP8JR1Rfo4OI/7/vDh7kq5D+TPdqRlmTkP3411GkQseQ/ZxGsGqpe5j+u0LjibLvnPyxACPwipug/l8KWkznc6D8eSq6bZEzqP+RtYqLvBus/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAAh/UAGfz7Q/pp1Q96jD7z8If1ABn8+0P0jIwmAibLY/VBAwuTgexT9082Ym4LnGP/zHBlIB5NM/yDqoOZxU1z9wjsxQryHYPywCf0bjKds/DGKCm0uP2z/T0WvkphDgPyQSkgRM5eA/QS/QoRwb4z8kSFMZYirjP6vcDemue+M/CLvaUOUA6z9IJlRMcQPrP21yT9NqVus/+t2pLFX+7D+sf8VbdqHvP6adUPeow+8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 4 AgMIEIAAAAAUAAAAAAAAABCw34Dy1K8/uPOLHG577z8QsN+A8tSvP4AQ2sMyJLA/mISIayLNuj+Y6ag0H929P0a7OGG0XdE/HJt+OU8s1T9SIQJtQrTVPyh39PxDRd8/BANiYb+93z8vDDgddOngPwFE0u6A9eI/TF4isMEU4z/3PkrhjZ7jPyzqqEfT9OM/gL9bQw1H5D8tHPixwtblP1u+aGcyAek/J10XftBo7T/RgJ6eVAruP7jzixxue+8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 4 AgMIEIAAAAAUAAAAAAAAADjwYBq6KbI/gYsDGJvg7z848GAauimyPxB6fe5KRsk/YkLSfV6F0T94q8WPqOjSPxRYUFXQmtQ/qDBlwyCX2T8Wruv2SmHbP4Th2rwN2ts/zGZBf1oP3j/EwoWDsI3hP5Ng8QFo0OM/ymr/Eic15D+4SOaMi9nlP9IvSvz6+eU/uz3wVfDu5j8T+yfhcPPnP8v2C0Cpfuk/z/PxPRyG6T9sxltQ9pPqP4GLAxib4O8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 1 AgMIEIAAAAAUAAAAAAAAAEDo189Vt6A/UM9Cl7Dm7z9A6NfPVbegP5ha69bbD7A/iM7IRCZTvj/4YFU7rSfAP7CmPAZDpMU/6MdLwA220j82NnAaWhzVPx6klGq+Qto/SLmxT+7G3T8aiwiNFBbgP2PbMYolm+E/s5xt06Jw4z+xkG5o9LbjPyCLL7eOOeY/l74V/0Ex7D/EFnjBE1HsP0VHTOcyWe0/kWCz4CQa7j8zN5zNh+XuP1DPQpew5u8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 6 AgMIEIAAAAAUAAAAAAAAAIBqSbZC4HM//Ai0w9as7z+Aakm2QuBzP2A1A5Qu35w/oGP25EuIsz9Yz0iumZK8PyAipOadlMk/poazYsyA1j9swUqKhePYP25v9bjTW9s/hEOeatZo2z8gthEWaHHeP605XJ6N0uE/w+MGYHmg5D+JiVd2jafmPxOhq3YMrec/9a9h4DFL6T/SHFHUZ4TrP9esDC6YK+w/klvwkl6l7D9UAc2LXhfuP/wItMPWrO8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 3 AgMIEIAAAAAUAAAAAAAAAMCKxfn9FpM/3a1pNLM57z/AisX5/RaTP2AslXZKxaE/IACvfHgFvT+8TKYiOXfMPxSFcOSIbdI/oj8xt0Ki1T/GZEADR/PXP8g37R5xRdo/BFpxoDB22j9aVglB2v3aP1q9rfHPft4/zl/bBz7H4D8kBWASkTHiP4uT+vqQ0+I/i73Yc1/l4j8vFE3v7oTmP6Og2m7Z8Og/x27veeKk7D98EKH4YjLtP92taTSzOe8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 4 AgMIEIAAAAAUAAAAAAAAAPCvi6n37ao/W3jYcspc7z/wr4up9+2qP9jJ9zxtJLI/EAGaiHz3tT/wkikAdcu3P5Ds1PcgA8c/SrG9mBhJ0T/8ry6B9Q/UPxD1ZTq2mdQ/4G9tlnq13z/Ut3XGVb7fPwgVwlgXI+E/9HItq+2U4z80GQKY6AzlP/CWL1v13uU/h8lG7sz95z+ZrJRQDWnoP9JCHQGlOew/Pd+Y3G1c7T+YfmBEWffuP1t42HLKXO8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAADAvstN0CKo/Fi/8Q8b17T8wL7LTdAiqP4AnASyh0Ls/IIbcCdolvj+IAG4861nFPzBYy5ojdMo/7ISSzuS4zT/IdkLG+ZXcP7LK4ib3J94/HB0Av7CP3j+LBaVhFQXgP3UE9JzC8eI/CZprMhIX4z+jeUrP/o7mP+NWnC9kkeY/0B7qM3ev5j+SBzE1C0PnP9i+KPoRF+o/S4xh/5ss7D9YNRmoM+XtPxYv/EPG9e0/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 1 AgMIEIAAAAAUAAAAAAAAAECSmXCRI44/xH/yfd3f7z9AkplwkSOOP+C4C/Im56U/cFWv+QBrrT9gyq70BP25P7DV2eoEg7w/YMnBhQOQzT+4Tzp2KLzNP3wq4CxHJ9Q/jnmjcfv92T9oTB66zDXcP0hTwIIgUt0/nDnm7BXX3T9ev8s1guviPyvI92A6QuQ/U/iauTcu5T9s4D2SFPvmP7wcRk1YsOg/pz9Tqf0m6j/ZdTscp9juP8R/8n3d3+8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 2 AgMIEIAAAAAUAAAAAAAAAIAHyxXav3k/I972lGXG7T+AB8sV2r95P4A7s5bBmoE/rHC32fK1wT841hadadPTP/BoIumS1NM/vk73/Pcy1D9MlU8W0qXXP/xbPpnTfdg/UA8cUeUA3D/evN37c33cP8bEFkSLxuI/Uhvc5Ptq4z9gSGQo3brlP8UcQgelF+Y/lGAi8Oyr5j/Fq4JOeZDoP6aedfj8Nuo/looj9vqe7D/vpBHUdsPsPyPe9pRlxu0/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAEBWpERB7o8/beu464Zp7j9AVqREQe6PP4Ab9nsi3qo/UMVzwdc4rz9ojiLWsya8P0yxpL26hsc/CDgR4XfYyj86Zms9otvVP+gxj0+uA9Y/lDti0Dz81j/W+iWyVYfaPxhPJAkk1No/FPQ+4MVw3T8vAjqDLHngPyLkmQcuuuA/y4CwvIrE4T8uZsd1KSLjP/GxZsdIROY//NevvkI85z910QE1bYDtP23ruOuGae4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 0 AgMIEIAAAAAUAAAAAAAAAOQazM3oz8g/th9Rd0Vu7D/kGszN6M/IP8wjLnWqTM8/wkIH6uZz2D9SE7AQEoTYPzBZ5xsbw94/Ogejduhz4j9EOCjBEAPjP0F7atUeC+M/3Mjise3j5T/CJc/6P4zmP9hEv68Vs+Y/e6HmmE016D8+MrjBWw3pPyZjrdR4cek/rAu5R1/t6T+PkGyBh/PpP9hT8TvccOo/E46awzgt6z8XTyf+iITrP7YfUXdFbuw/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 7 AgMIEIAAAAAUAAAAAAAAAAA64t2+TFI/BJnMnZCp7j8AOuLdvkxSPwCoIjvLCHY/YOfQY4T5tD/Uop4ic9vCPySDfUNEysc/tuBfgSbo1T/Mfn4/s8nXP3Ii+O/nktg/HpCvdatc3D+ULFcJfRLeP1Z8xIRR3t4/lxQP8ipO4z9gBobpAnbjP7x6XqanhOQ/hxrHzpCI5T9gYwrf5mPnP/fQyzNLqOk/AsjhBDwy6z80a1ADxbDsPwSZzJ2Qqe4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAIDHPwsa7oM/NOu2UYOC7z+Axz8LGu6DP0Cls/BvKZg/SD9mURUiuj+kNll7CvbMP3qiiycHCNI/oDEZs8j30z8GmW4gajTUPwj4arsVstg/LjtkY3B82T+QkuK7B+vaP1i82/6vEeE/9kY5bylg4j+uM10lQBvjPxw16z7/CuU/UgBmDzkc6T+WOmPkhHXrPxaSMxzJl+4/H6m+hlnK7j+7qat3Ym7vPzTrtlGDgu8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAGAMeSH7cZg/mZh2OEfG7j9gDHkh+3GYP4DVUtxEqr8/uGSsRILnzz+QYqh2i/rSPxgyZML0k9M/TLvGaRrd0z+qUlTIrT7WP0jwnIsU69Y/9DlbuJQV1z80odEVCi7XPzYJAWptlNg/xn5A3lZF2j/8IYNmyJXfP9/zj0NcF+A/J7hrIQXQ4T9er8p9YxnjPykVGmSMnOQ/HupHGIYM7D9Ctalfdy/tP5mYdjhHxu4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 6 AgMIEIAAAAAUAAAAAAAAAGCaMPB7uaA/vm3Zk6tj7j9gmjDwe7mgPwAyraIwZ6I/wLvfORBkvz9s4m2/XbzJP/h2CoHxAdM/JlMAz4pd1T9ELRFW2mLWP1IMxQZMbNg/0gPwr12m2D8RAX8l30fgP7ZZjdsZ5+E/Py0fVQo14j+aXxPHhcPiP7jvIjXrduQ/sHkdBYYW5T/+kmIjwRvlPxwTolKR7uc/QwfgNDdS6z9oY+HuTFntP75t2ZOrY+4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 9 AgMIEIAAAAAUAAAAAAAAAPDVihP0v7U/ym6PAbOK7j/w1YoT9L+1P6CkQb6Yw7k/ECDaOoIvyD9QThdPIQDLP/Kx1PzYQNA/PPMQDwDY0D/AOShK0t3SPzakozAiLNc/+tcJbK2F2j9opwh3SAXbP9DKhtW6CN4/XzOJ51Xv4D+nujSMUIzhPz2LFONvGuU//OfzStrm6D+8JasYgGPpP3Iwmygz6eo/9q80a4G66z+30ydchNrsP8pujwGziu4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAAhHLGcu8Lc/GCufaRJs7j8IRyxnLvC3P9DoEP7eibo/kEm1ijvzxD8IFZE1i3nKP2S3+uNk+c8/vAAC9FR40D8UyZlapsbRPyTa0i4O+dY/PP61QdvF2T8yu5kc2P3bP+76ADRYrtw/8bPIe9ma5D9q8YTupGLmP3jPYpQY8uY/O8XQ1ZTj6z/wa4p9Y6bsP4wdz433He0/QudyTYrc7T+Gy8vXYR3uPxgrn2kSbO4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== From b6eae594a6be154ffd06dbce4e1b353f595b4acf Mon Sep 17 00:00:00 2001 From: saydakov Date: Tue, 24 Oct 2017 16:01:33 -0700 Subject: [PATCH 02/17] it seems that we need to synchronize all methods, which modify the state --- .../datasketches/quantiles/DoublesSketchAggregator.java | 6 +++--- .../quantiles/DoublesSketchBufferAggregator.java | 8 ++++---- .../quantiles/DoublesSketchDoubleAggregator.java | 6 +++--- .../quantiles/DoublesSketchDoubleBufferAggregator.java | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java index 553b49abf25e..403418b6f49d 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java @@ -48,7 +48,7 @@ public synchronized void aggregate() } @Override - public Object get() + public synchronized Object get() { return union.getResult(); } @@ -66,13 +66,13 @@ public long getLong() } @Override - public void reset() + public synchronized void reset() { union.reset(); } @Override - public void close() + public synchronized void close() { union = null; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java index 20e0ab5a5599..5125811b064b 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java @@ -52,7 +52,7 @@ public DoublesSketchBufferAggregator( } @Override - public void init(final ByteBuffer buffer, final int position) + public synchronized void init(final ByteBuffer buffer, final int position) { final WritableMemory mem = getMemory(buffer); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); @@ -72,7 +72,7 @@ public synchronized void aggregate(final ByteBuffer buffer, final int position) } @Override - public Object get(final ByteBuffer buffer, final int position) + public synchronized Object get(final ByteBuffer buffer, final int position) { return unions.get(buffer).get(position).getResult(); } @@ -90,14 +90,14 @@ public long getLong(final ByteBuffer buffer, final int position) } @Override - public void close() + public synchronized void close() { unions.clear(); memCache.clear(); } @Override - public void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) + public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) { DoublesUnion union = unions.get(oldBuffer).get(oldPosition); final WritableMemory oldMem = getMemory(oldBuffer).writableRegion(oldPosition, maxIntermediateSize); diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java index 5cf2d8f0a61c..6b2659a5c780 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java @@ -46,7 +46,7 @@ public synchronized void aggregate() } @Override - public Object get() + public synchronized Object get() { return sketch; } @@ -64,13 +64,13 @@ public long getLong() } @Override - public void reset() + public synchronized void reset() { sketch = UpdateDoublesSketch.builder().setK(size).build(); } @Override - public void close() + public synchronized void close() { sketch = null; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java index e10b6ee9bbd0..76108532eb8c 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java @@ -50,7 +50,7 @@ public DoublesSketchDoubleBufferAggregator(final ColumnValueSelector val } @Override - public void init(final ByteBuffer buffer, final int position) + public synchronized void init(final ByteBuffer buffer, final int position) { final WritableMemory mem = getMemory(buffer); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); @@ -66,7 +66,7 @@ public synchronized void aggregate(final ByteBuffer buffer, final int position) } @Override - public Object get(final ByteBuffer buffer, final int position) + public synchronized Object get(final ByteBuffer buffer, final int position) { return sketches.get(buffer).get(position); } @@ -84,14 +84,14 @@ public long getLong(final ByteBuffer buffer, final int position) } @Override - public void close() + public synchronized void close() { sketches.clear(); memCache.clear(); } @Override - public void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) + public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) { UpdateDoublesSketch sketch = sketches.get(oldBuffer).get(oldPosition); final WritableMemory oldRegion = getMemory(oldBuffer).writableRegion(oldPosition, maxIntermediateSize); From 826a54e94d8a3bab9f9ae22f9ba79a2eba6fac1a Mon Sep 17 00:00:00 2001 From: saydakov Date: Wed, 25 Oct 2017 10:54:07 -0700 Subject: [PATCH 03/17] Seems like a false positive with -Pstrict --- .../aggregation/datasketches/theta/SynchronizedUnion.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java index 6fa21fec6bbe..41e3d8ee410a 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java @@ -68,18 +68,21 @@ public synchronized void update(String datum) } @Override + @SuppressWarnings("ParameterPackage") public synchronized void update(byte[] data) { delegate.update(data); } @Override + @SuppressWarnings("ParameterPackage") public synchronized void update(int[] data) { delegate.update(data); } @Override + @SuppressWarnings("ParameterPackage") public synchronized void update(char[] chars) { delegate.update(chars); From 5b64b523dd4c61851408f56a093e357e5c3c5c68 Mon Sep 17 00:00:00 2001 From: saydakov Date: Mon, 30 Oct 2017 12:54:58 -0700 Subject: [PATCH 04/17] code style fix --- .../DoublesSketchAggregatorTest.java | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java index 45ceb6e258ac..f0b0ea210034 100644 --- a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java @@ -68,7 +68,7 @@ public static Collection constructorFeeder() { final List constructors = Lists.newArrayList(); for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { - constructors.add(new Object[] {config}); + constructors.add(new Object[] { config }); } return constructors; } @@ -156,10 +156,9 @@ public void ingestingSketches() throws Exception Object histogramObject = row.getRaw("histogram"); Assert.assertTrue(histogramObject instanceof double[]); double[] histogram = (double[]) histogramObject; - for (int i = 0; i < histogram.length; i++) { - Assert.assertEquals(100, histogram[i], 100 * 0.2); // 400 items uniformly - // distributed into 4 - // bins + for (final double bin: histogram) { + Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly + // distributed into 4 bins } } @@ -224,10 +223,9 @@ public void buildingSketchesAtIngestionTime() throws Exception Assert.assertTrue(histogramObject instanceof double[]); double[] histogram = (double[]) histogramObject; Assert.assertEquals(4, histogram.length); - for (int i = 0; i < histogram.length; i++) { - Assert.assertEquals(100, histogram[i], 100 * 0.2); // 400 items uniformly - // distributed into 4 - // bins + for (final double bin: histogram) { + Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly + // distributed into 4 bins } } @@ -296,10 +294,9 @@ public void buildingSketchesAtQueryTime() throws Exception Object histogramObject = row.getRaw("histogram"); Assert.assertTrue(histogramObject instanceof double[]); double[] histogram = (double[]) histogramObject; - for (int i = 0; i < histogram.length; i++) { - Assert.assertEquals(100, histogram[i], 100 * 0.2); // 400 items uniformly - // distributed into 4 - // bins + for (final double bin: histogram) { + Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly + // distributed into 4 bins } } @@ -370,10 +367,9 @@ public void QueryingDataWithFieldNameValueAsFloatInsteadOfSketch() throws Except Object histogramObject = row.getRaw("histogram"); Assert.assertTrue(histogramObject instanceof double[]); double[] histogram = (double[]) histogramObject; - for (int i = 0; i < histogram.length; i++) { - Assert.assertEquals(100, histogram[i], 100 * 0.2); // 400 items uniformly - // distributed into 4 - // bins + for (final double bin: histogram) { + Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly + // distributed into 4 bins } } From a2d0396efcdcf2611e0d16b7924f19054c857e35 Mon Sep 17 00:00:00 2001 From: saydakov Date: Mon, 30 Oct 2017 14:32:20 -0700 Subject: [PATCH 05/17] code style fix --- .../datasketches/quantiles/DoublesSketchAggregatorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java index f0b0ea210034..899c8392e4fd 100644 --- a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java @@ -68,7 +68,7 @@ public static Collection constructorFeeder() { final List constructors = Lists.newArrayList(); for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { - constructors.add(new Object[] { config }); + constructors.add(new Object[] {config}); } return constructors; } From e57a870c2286702cf37fe33a46eac368146c9945 Mon Sep 17 00:00:00 2001 From: saydakov Date: Mon, 30 Oct 2017 15:20:53 -0700 Subject: [PATCH 06/17] use sketches-core-0.10.3 --- extensions-core/datasketches/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 974a19fcb430..f16fa37ec795 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -38,7 +38,7 @@ com.yahoo.datasketches sketches-core - 0.10.2 + 0.10.3 io.druid From 65e193d410755fc5f52a60c99a2a776aee1c718b Mon Sep 17 00:00:00 2001 From: saydakov Date: Wed, 1 Nov 2017 12:00:04 -0700 Subject: [PATCH 07/17] moved cache ids to the central place --- .../CombiningDoublesSketchAggregatorFactory.java | 5 ++--- .../quantiles/DoublesSketchAggregatorFactory.java | 4 ++-- .../quantiles/DoublesSketchHistogramPostAggregator.java | 6 +++--- .../quantiles/DoublesSketchQuantilePostAggregator.java | 6 +++--- .../quantiles/DoublesSketchQuantilesPostAggregator.java | 6 +++--- .../quantiles/DoublesSketchToStringPostAggregator.java | 6 +++--- .../java/io/druid/query/aggregation/AggregatorUtil.java | 8 ++++++++ 7 files changed, 24 insertions(+), 17 deletions(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java index 841abf9759bc..3f7507c84d3e 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.sketches.quantiles.DoublesSketch; import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; @@ -30,14 +31,12 @@ public class CombiningDoublesSketchAggregatorFactory extends DoublesSketchAggregatorFactory { - private static final byte CACHE_TYPE_ID = 111; - @JsonCreator public CombiningDoublesSketchAggregatorFactory( @JsonProperty("name") final String name, @JsonProperty("k") final Integer k) { - super(name, name, k, CACHE_TYPE_ID); + super(name, name, k, AggregatorUtil.QUANTILES_DOUBLES_SKETCH_MERGE_CACHE_TYPE_ID); } @Override diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index ab5c35f9a87c..ec3d775addef 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -29,6 +29,7 @@ import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorFactoryNotMergeableException; +import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; import io.druid.query.cache.CacheKeyBuilder; import io.druid.segment.ColumnSelectorFactory; @@ -43,7 +44,6 @@ public class DoublesSketchAggregatorFactory extends AggregatorFactory { private static final int DEFAULT_K = 128; - private static final byte CACHE_TYPE_ID = 110; // Used for sketch size estimation. private static final long MAX_STREAM_LENGTH = 1_000_000_000; @@ -59,7 +59,7 @@ public DoublesSketchAggregatorFactory( @JsonProperty("fieldName") final String fieldName, @JsonProperty("k") final Integer k) { - this(name, fieldName, k, CACHE_TYPE_ID); + this(name, fieldName, k, AggregatorUtil.QUANTILES_DOUBLES_SKETCH_BUILD_CACHE_TYPE_ID); } DoublesSketchAggregatorFactory(final String name, final String fieldName, final Integer k, final byte cacheTypeId) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java index 911b2ba62769..add7e085c373 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; @@ -37,8 +38,6 @@ public class DoublesSketchHistogramPostAggregator implements PostAggregator { - private static final byte CACHE_KEY = 112; - private final String name; private final PostAggregator field; private final double[] splitPoints; @@ -144,7 +143,8 @@ public int hashCode() @Override public byte[] getCacheKey() { - final CacheKeyBuilder builder = new CacheKeyBuilder(CACHE_KEY).appendCacheable(field); + final CacheKeyBuilder builder = new CacheKeyBuilder( + AggregatorUtil.QUANTILES_DOUBLES_SKETCH_TO_HISTOGRAM_CACHE_TYPE_ID).appendCacheable(field); for (final double value: splitPoints) { builder.appendDouble(value); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java index 9ee94e28c1f1..7671474aa4ba 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java @@ -29,6 +29,7 @@ import com.google.common.primitives.Doubles; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; @@ -37,8 +38,6 @@ public class DoublesSketchQuantilePostAggregator implements PostAggregator { - private static final byte CACHE_KEY = 113; - private final String name; private final PostAggregator field; private final double fraction; @@ -137,7 +136,8 @@ public int hashCode() @Override public byte[] getCacheKey() { - return new CacheKeyBuilder(CACHE_KEY).appendCacheable(field).appendDouble(fraction).build(); + return new CacheKeyBuilder(AggregatorUtil.QUANTILES_DOUBLES_SKETCH_TO_QUANTILE_CACHE_TYPE_ID) + .appendCacheable(field).appendDouble(fraction).build(); } @Override diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java index 35f872fe4ad3..a08bda284b52 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; @@ -37,8 +38,6 @@ public class DoublesSketchQuantilesPostAggregator implements PostAggregator { - private static final byte CACHE_KEY = 114; - private final String name; private final PostAggregator field; private final double[] fractions; @@ -139,7 +138,8 @@ public int hashCode() @Override public byte[] getCacheKey() { - final CacheKeyBuilder builder = new CacheKeyBuilder(CACHE_KEY).appendCacheable(field); + final CacheKeyBuilder builder = new CacheKeyBuilder( + AggregatorUtil.QUANTILES_DOUBLES_SKETCH_TO_QUANTILES_CACHE_TYPE_ID).appendCacheable(field); for (final double value: fractions) { builder.appendDouble(value); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java index 6c56dfdd2d7c..ceeb7d41ece1 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java @@ -28,6 +28,7 @@ import com.google.common.base.Preconditions; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; @@ -36,8 +37,6 @@ public class DoublesSketchToStringPostAggregator implements PostAggregator { - private static final byte CACHE_KEY = 115; - private final String name; private final PostAggregator field; @@ -125,7 +124,8 @@ public int hashCode() @Override public byte[] getCacheKey() { - final CacheKeyBuilder builder = new CacheKeyBuilder(CACHE_KEY).appendCacheable(field); + final CacheKeyBuilder builder = new CacheKeyBuilder( + AggregatorUtil.QUANTILES_DOUBLES_SKETCH_TO_STRING_CACHE_TYPE_ID).appendCacheable(field); return builder.build(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 895f1056f966..fc3ffaee77b7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -73,6 +73,14 @@ public class AggregatorUtil public static final byte TIMESTAMP_CACHE_TYPE_ID = 0x19; public static final byte VARIANCE_CACHE_TYPE_ID = 0x1A; + // Quantiles sketch aggregator + public static final byte QUANTILES_DOUBLES_SKETCH_BUILD_CACHE_TYPE_ID = 0x1B; + public static final byte QUANTILES_DOUBLES_SKETCH_MERGE_CACHE_TYPE_ID = 0x1C; + public static final byte QUANTILES_DOUBLES_SKETCH_TO_HISTOGRAM_CACHE_TYPE_ID = 0x1D; + public static final byte QUANTILES_DOUBLES_SKETCH_TO_QUANTILE_CACHE_TYPE_ID = 0x1E; + public static final byte QUANTILES_DOUBLES_SKETCH_TO_QUANTILES_CACHE_TYPE_ID = 0x1F; + public static final byte QUANTILES_DOUBLES_SKETCH_TO_STRING_CACHE_TYPE_ID = 0x20; + /** * returns the list of dependent postAggregators that should be calculated in order to calculate given postAgg * From a2e0d7756e490cfc9aec17b3b4f29881b99dda66 Mon Sep 17 00:00:00 2001 From: saydakov Date: Thu, 2 Nov 2017 13:37:08 -0700 Subject: [PATCH 08/17] better class names --- .../DoublesSketchAggregatorFactory.java | 20 +++++++++---------- ...java => DoublesSketchBuildAggregator.java} | 4 ++-- ...> DoublesSketchBuildBufferAggregator.java} | 4 ++-- ...java => DoublesSketchMergeAggregator.java} | 4 ++-- ... DoublesSketchMergeAggregatorFactory.java} | 12 +++++------ ...> DoublesSketchMergeBufferAggregator.java} | 4 ++-- .../quantiles/DoublesSketchModule.java | 6 +++--- ....java => DoublesSketchNoOpAggregator.java} | 2 +- ...=> DoublesSketchNoOpBufferAggregator.java} | 2 +- ...ublesSketchToHistogramPostAggregator.java} | 6 +++--- ...oublesSketchToQuantilePostAggregator.java} | 6 +++--- ...ublesSketchToQuantilesPostAggregator.java} | 6 +++--- 12 files changed, 38 insertions(+), 38 deletions(-) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{DoublesSketchDoubleAggregator.java => DoublesSketchBuildAggregator.java} (91%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{DoublesSketchDoubleBufferAggregator.java => DoublesSketchBuildBufferAggregator.java} (95%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{DoublesSketchAggregator.java => DoublesSketchMergeAggregator.java} (91%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{CombiningDoublesSketchAggregatorFactory.java => DoublesSketchMergeAggregatorFactory.java} (83%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{DoublesSketchBufferAggregator.java => DoublesSketchMergeBufferAggregator.java} (97%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{EmptyDoublesSketchAggregator.java => DoublesSketchNoOpAggregator.java} (95%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{EmptyDoublesSketchBufferAggregator.java => DoublesSketchNoOpBufferAggregator.java} (95%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{DoublesSketchHistogramPostAggregator.java => DoublesSketchToHistogramPostAggregator.java} (94%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{DoublesSketchQuantilePostAggregator.java => DoublesSketchToQuantilePostAggregator.java} (94%) rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/{DoublesSketchQuantilesPostAggregator.java => DoublesSketchToQuantilesPostAggregator.java} (94%) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index ec3d775addef..261fcd80ca01 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -84,15 +84,15 @@ public Aggregator factorize(final ColumnSelectorFactory metricFactory) && ValueType.isNumeric(metricFactory.getColumnCapabilities(fieldName).getType())) { final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector == null) { - return new EmptyDoublesSketchAggregator(); + return new DoublesSketchNoOpAggregator(); } - return new DoublesSketchDoubleAggregator(valueSelector, k); + return new DoublesSketchBuildAggregator(valueSelector, k); } final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); if (selector == null) { - return new EmptyDoublesSketchAggregator(); + return new DoublesSketchNoOpAggregator(); } - return new DoublesSketchAggregator(selector, k); + return new DoublesSketchMergeAggregator(selector, k); } @Override @@ -104,17 +104,17 @@ public BufferAggregator factorizeBuffered(final ColumnSelectorFactory metricFact } final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); if (selector == null) { - return new EmptyDoublesSketchBufferAggregator(); + return new DoublesSketchNoOpBufferAggregator(); } - return new DoublesSketchBufferAggregator(selector, k, getMaxIntermediateSize()); + return new DoublesSketchMergeBufferAggregator(selector, k, getMaxIntermediateSize()); } private BufferAggregator getDoubleBufferAggregator(final ColumnValueSelector selector) { if (selector == null) { - return new EmptyDoublesSketchBufferAggregator(); + return new DoublesSketchNoOpBufferAggregator(); } - return new DoublesSketchDoubleBufferAggregator(selector, k, getMaxIntermediateSize()); + return new DoublesSketchBuildBufferAggregator(selector, k, getMaxIntermediateSize()); } @Override @@ -191,14 +191,14 @@ public List getRequiredColumns() @Override public AggregatorFactory getCombiningFactory() { - return new CombiningDoublesSketchAggregatorFactory(name, k); + return new DoublesSketchMergeAggregatorFactory(name, k); } @Override public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException { if (other.getName().equals(this.getName()) && other instanceof DoublesSketchAggregatorFactory) { - return new CombiningDoublesSketchAggregatorFactory(name, k); + return new DoublesSketchMergeAggregatorFactory(name, k); } else { throw new AggregatorFactoryNotMergeableException(this, other); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildAggregator.java similarity index 91% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildAggregator.java index 6b2659a5c780..f957f5bb0ce1 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildAggregator.java @@ -24,7 +24,7 @@ import io.druid.query.aggregation.Aggregator; import io.druid.segment.ColumnValueSelector; -public class DoublesSketchDoubleAggregator implements Aggregator +public class DoublesSketchBuildAggregator implements Aggregator { private final ColumnValueSelector valueSelector; @@ -32,7 +32,7 @@ public class DoublesSketchDoubleAggregator implements Aggregator private UpdateDoublesSketch sketch; - public DoublesSketchDoubleAggregator(final ColumnValueSelector valueSelector, final int size) + public DoublesSketchBuildAggregator(final ColumnValueSelector valueSelector, final int size) { this.valueSelector = valueSelector; this.size = size; diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java similarity index 95% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java index 76108532eb8c..9d735353ebe4 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchDoubleBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java @@ -31,7 +31,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; -public class DoublesSketchDoubleBufferAggregator implements BufferAggregator +public class DoublesSketchBuildBufferAggregator implements BufferAggregator { private final ColumnValueSelector selector; @@ -41,7 +41,7 @@ public class DoublesSketchDoubleBufferAggregator implements BufferAggregator private final IdentityHashMap memCache = new IdentityHashMap<>(); private final IdentityHashMap> sketches = new IdentityHashMap<>(); - public DoublesSketchDoubleBufferAggregator(final ColumnValueSelector valueSelector, final int size, + public DoublesSketchBuildBufferAggregator(final ColumnValueSelector valueSelector, final int size, final int maxIntermediateSize) { this.selector = valueSelector; diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java similarity index 91% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java index 403418b6f49d..6152ff417a34 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java @@ -25,13 +25,13 @@ import io.druid.query.aggregation.Aggregator; import io.druid.segment.ColumnValueSelector; -public class DoublesSketchAggregator implements Aggregator +public class DoublesSketchMergeAggregator implements Aggregator { private final ColumnValueSelector selector; private DoublesUnion union; - public DoublesSketchAggregator(final ColumnValueSelector selector, final int k) + public DoublesSketchMergeAggregator(final ColumnValueSelector selector, final int k) { this.selector = selector; union = DoublesUnion.builder().setMaxK(k).build(); diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java similarity index 83% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java index 3f7507c84d3e..110e8a8523e2 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/CombiningDoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java @@ -28,11 +28,11 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; -public class CombiningDoublesSketchAggregatorFactory extends DoublesSketchAggregatorFactory +public class DoublesSketchMergeAggregatorFactory extends DoublesSketchAggregatorFactory { @JsonCreator - public CombiningDoublesSketchAggregatorFactory( + public DoublesSketchMergeAggregatorFactory( @JsonProperty("name") final String name, @JsonProperty("k") final Integer k) { @@ -44,9 +44,9 @@ public Aggregator factorize(final ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); if (selector == null) { - return new EmptyDoublesSketchAggregator(); + return new DoublesSketchNoOpAggregator(); } - return new DoublesSketchAggregator(selector, getK()); + return new DoublesSketchMergeAggregator(selector, getK()); } @Override @@ -54,9 +54,9 @@ public BufferAggregator factorizeBuffered(final ColumnSelectorFactory metricFact { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); if (selector == null) { - return new EmptyDoublesSketchBufferAggregator(); + return new DoublesSketchNoOpBufferAggregator(); } - return new DoublesSketchBufferAggregator(selector, getK(), getMaxIntermediateSize()); + return new DoublesSketchMergeBufferAggregator(selector, getK(), getMaxIntermediateSize()); } } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java similarity index 97% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java index 5125811b064b..9619bd5fcd03 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java @@ -32,7 +32,7 @@ import java.nio.ByteBuffer; import java.util.IdentityHashMap; -public class DoublesSketchBufferAggregator implements BufferAggregator +public class DoublesSketchMergeBufferAggregator implements BufferAggregator { private final ColumnValueSelector selector; @@ -41,7 +41,7 @@ public class DoublesSketchBufferAggregator implements BufferAggregator private final IdentityHashMap memCache = new IdentityHashMap<>(); private final IdentityHashMap> unions = new IdentityHashMap<>(); - public DoublesSketchBufferAggregator( + public DoublesSketchMergeBufferAggregator( final ColumnValueSelector selector, final int k, final int maxIntermediateSize) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java index c988bcefbe32..0e3081965b03 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java @@ -55,9 +55,9 @@ public List getJacksonModules() return Arrays. asList( new SimpleModule("DoublesQuantilesSketchModule").registerSubtypes( new NamedType(DoublesSketchAggregatorFactory.class, DOUBLES_SKETCH), - new NamedType(DoublesSketchHistogramPostAggregator.class, DOUBLES_SKETCH_HISTOGRAM_POST_AGG), - new NamedType(DoublesSketchQuantilePostAggregator.class, DOUBLES_SKETCH_QUANTILE_POST_AGG), - new NamedType(DoublesSketchQuantilesPostAggregator.class, DOUBLES_SKETCH_QUANTILES_POST_AGG), + new NamedType(DoublesSketchToHistogramPostAggregator.class, DOUBLES_SKETCH_HISTOGRAM_POST_AGG), + new NamedType(DoublesSketchToQuantilePostAggregator.class, DOUBLES_SKETCH_QUANTILE_POST_AGG), + new NamedType(DoublesSketchToQuantilesPostAggregator.class, DOUBLES_SKETCH_QUANTILES_POST_AGG), new NamedType(DoublesSketchToStringPostAggregator.class, DOUBLES_SKETCH_TO_STRING_POST_AGG)) .addSerializer(DoublesSketch.class, new DoublesSketchJsonSerializer())); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpAggregator.java similarity index 95% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpAggregator.java index efb024d61036..2cfeb03a534b 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpAggregator.java @@ -21,7 +21,7 @@ import io.druid.query.aggregation.Aggregator; -public class EmptyDoublesSketchAggregator implements Aggregator +public class DoublesSketchNoOpAggregator implements Aggregator { @Override diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpBufferAggregator.java similarity index 95% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchBufferAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpBufferAggregator.java index 4c34f787ad15..ad61ff5372f8 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/EmptyDoublesSketchBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpBufferAggregator.java @@ -24,7 +24,7 @@ import io.druid.query.aggregation.BufferAggregator; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -public class EmptyDoublesSketchBufferAggregator implements BufferAggregator +public class DoublesSketchNoOpBufferAggregator implements BufferAggregator { @Override diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java similarity index 94% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java index add7e085c373..b3f862f0da7d 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchHistogramPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java @@ -35,7 +35,7 @@ import com.yahoo.sketches.quantiles.DoublesSketch; -public class DoublesSketchHistogramPostAggregator implements PostAggregator +public class DoublesSketchToHistogramPostAggregator implements PostAggregator { private final String name; @@ -43,7 +43,7 @@ public class DoublesSketchHistogramPostAggregator implements PostAggregator private final double[] splitPoints; @JsonCreator - public DoublesSketchHistogramPostAggregator( + public DoublesSketchToHistogramPostAggregator( @JsonProperty("name") final String name, @JsonProperty("field") final PostAggregator field, @JsonProperty("splitPoints") final double[] splitPoints) @@ -122,7 +122,7 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final DoublesSketchHistogramPostAggregator that = (DoublesSketchHistogramPostAggregator) o; + final DoublesSketchToHistogramPostAggregator that = (DoublesSketchToHistogramPostAggregator) o; if (!name.equals(that.name)) { return false; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java similarity index 94% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java index 7671474aa4ba..9aacc8bb005c 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java @@ -35,7 +35,7 @@ import com.yahoo.sketches.quantiles.DoublesSketch; -public class DoublesSketchQuantilePostAggregator implements PostAggregator +public class DoublesSketchToQuantilePostAggregator implements PostAggregator { private final String name; @@ -43,7 +43,7 @@ public class DoublesSketchQuantilePostAggregator implements PostAggregator private final double fraction; @JsonCreator - public DoublesSketchQuantilePostAggregator( + public DoublesSketchToQuantilePostAggregator( @JsonProperty("name") final String name, @JsonProperty("field") final PostAggregator field, @JsonProperty("fraction") final double fraction) @@ -117,7 +117,7 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final DoublesSketchQuantilePostAggregator that = (DoublesSketchQuantilePostAggregator) o; + final DoublesSketchToQuantilePostAggregator that = (DoublesSketchToQuantilePostAggregator) o; if (!name.equals(that.name)) { return false; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java similarity index 94% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java index a08bda284b52..2da179ecba4c 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchQuantilesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java @@ -35,7 +35,7 @@ import com.yahoo.sketches.quantiles.DoublesSketch; -public class DoublesSketchQuantilesPostAggregator implements PostAggregator +public class DoublesSketchToQuantilesPostAggregator implements PostAggregator { private final String name; @@ -43,7 +43,7 @@ public class DoublesSketchQuantilesPostAggregator implements PostAggregator private final double[] fractions; @JsonCreator - public DoublesSketchQuantilesPostAggregator( + public DoublesSketchToQuantilesPostAggregator( @JsonProperty("name") final String name, @JsonProperty("field") final PostAggregator field, @JsonProperty("fractions") final double[] fractions) @@ -119,7 +119,7 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final DoublesSketchQuantilesPostAggregator that = (DoublesSketchQuantilesPostAggregator) o; + final DoublesSketchToQuantilesPostAggregator that = (DoublesSketchToQuantilesPostAggregator) o; if (!name.equals(that.name)) { return false; } From 3f23722c2e4517c1ab384d62099cd19aa003ca9e Mon Sep 17 00:00:00 2001 From: saydakov Date: Thu, 2 Nov 2017 14:51:16 -0700 Subject: [PATCH 09/17] support large columns --- .../quantiles/DoublesSketchComplexMetricSerde.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java index da81f135923c..463c974f606f 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java @@ -23,12 +23,15 @@ import com.yahoo.sketches.quantiles.DoublesSketch; import com.yahoo.sketches.quantiles.UpdateDoublesSketch; import io.druid.data.input.InputRow; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.serde.ComplexColumnPartSupplier; import io.druid.segment.serde.ComplexMetricExtractor; import io.druid.segment.serde.ComplexMetricSerde; +import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; import java.nio.ByteBuffer; @@ -91,8 +94,15 @@ public Object extractValue(final InputRow inputRow, final String metricName) @Override public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder) { - final GenericIndexed column = GenericIndexed.read(buffer, strategy); + final GenericIndexed column = GenericIndexed.read(buffer, strategy, builder.getFileMapper()); builder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); } + // support large columns + @Override + public GenericColumnSerializer getSerializer(IOPeon peon, String column) + { + return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy()); + } + } From 7c9d0b682c065f074bc4591138cfc89070aef216 Mon Sep 17 00:00:00 2001 From: saydakov Date: Thu, 2 Nov 2017 15:43:14 -0700 Subject: [PATCH 10/17] explained autodetection, added exception --- .../quantiles/DoublesSketchComplexMetricSerde.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java index 463c974f606f..afd4f933a0df 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java @@ -23,6 +23,7 @@ import com.yahoo.sketches.quantiles.DoublesSketch; import com.yahoo.sketches.quantiles.UpdateDoublesSketch; import io.druid.data.input.InputRow; +import io.druid.java.util.common.IAE; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; @@ -69,6 +70,9 @@ public Object extractValue(final InputRow inputRow, final String metricName) final Object object = inputRow.getRaw(metricName); if (object instanceof String) { String objectString = (String) object; + // Autodetection of the input format: a number or base64 encoded sketch + // A serialized DoublesSketch, as currently implemented, always has 0 in the first 6 bits. + // This corresponds to "A" in base64, so it is not a digit if (Character.isDigit((objectString).charAt(0))) { try { Double doubleValue = Double.parseDouble(objectString); @@ -77,8 +81,7 @@ public Object extractValue(final InputRow inputRow, final String metricName) return sketch; } catch (NumberFormatException e) { - // Log.info("Expected Double. Got string with value " + - // objectString); + throw new IAE("Expected a string with a number, received value " + objectString); } } } From 7e1502903afde32267df3a7fe29ea74d20943f89 Mon Sep 17 00:00:00 2001 From: saydakov Date: Thu, 2 Nov 2017 15:59:46 -0700 Subject: [PATCH 11/17] added comments regarding sketches moving on heap --- .../datasketches/quantiles/DoublesSketchAggregatorFactory.java | 3 +++ .../quantiles/DoublesSketchBuildBufferAggregator.java | 2 ++ .../quantiles/DoublesSketchMergeBufferAggregator.java | 2 ++ 3 files changed, 7 insertions(+) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index 261fcd80ca01..b0ccd5be91ec 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -172,6 +172,9 @@ public List requiredFields() return Collections.singletonList(fieldName); } + // Quantiles sketches never stop growing, but they do so very slowly. + // This size must suffice for overwhelming majority of sketches, + // but some sketches may request more memory on heap and move there @Override public int getMaxIntermediateSize() { diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java index 9d735353ebe4..54b2edfe2cba 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java @@ -90,6 +90,8 @@ public synchronized void close() memCache.clear(); } + // A small number of sketches may run out of the given memory, request more memory on heap and move there. + // In that case we need to reuse the object from the cache as opposed to wrapping the new buffer. @Override public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) { diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java index 9619bd5fcd03..3a7d5e842c43 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java @@ -96,6 +96,8 @@ public synchronized void close() memCache.clear(); } + // A small number of sketches may run out of the given memory, request more memory on heap and move there. + // In that case we need to reuse the object from the cache as opposed to wrapping the new buffer. @Override public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) { From cacd6a54700542388bbb18a7283a506d080260d1 Mon Sep 17 00:00:00 2001 From: saydakov Date: Fri, 17 Nov 2017 12:59:49 -0800 Subject: [PATCH 12/17] support reindexing --- .../quantiles/DoublesSketchComplexMetricSerde.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java index afd4f933a0df..a23fdb822b05 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java @@ -68,7 +68,7 @@ public Class extractedClass() public Object extractValue(final InputRow inputRow, final String metricName) { final Object object = inputRow.getRaw(metricName); - if (object instanceof String) { + if (object instanceof String) { // everything is a string during ingestion String objectString = (String) object; // Autodetection of the input format: a number or base64 encoded sketch // A serialized DoublesSketch, as currently implemented, always has 0 in the first 6 bits. @@ -84,6 +84,10 @@ public Object extractValue(final InputRow inputRow, final String metricName) throw new IAE("Expected a string with a number, received value " + objectString); } } + } else if (object instanceof Number) { // this is for reindexing + UpdateDoublesSketch sketch = DoublesSketch.builder().setK(2).build(); + sketch.update(((Number) object).doubleValue()); + return sketch; } if (object == null || object instanceof DoublesSketch || object instanceof Memory) { From 84a9c793d449326dad31e0e032f60fe263ae690e Mon Sep 17 00:00:00 2001 From: saydakov Date: Mon, 20 Nov 2017 16:03:09 -0800 Subject: [PATCH 13/17] implemented suggestions from jihoonson --- .../quantiles/DoublesSketchAggregatorFactory.java | 12 ++++++------ .../DoublesSketchBuildBufferAggregator.java | 13 ++----------- .../quantiles/DoublesSketchComplexMetricSerde.java | 6 ++++-- .../quantiles/DoublesSketchJsonSerializer.java | 3 +-- .../DoublesSketchMergeBufferAggregator.java | 13 ++----------- .../quantiles/DoublesSketchOperations.java | 4 +++- .../datasketches/quantiles/GenerateTestData.java | 1 + 7 files changed, 19 insertions(+), 33 deletions(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index b0ccd5be91ec..e482e54f0aa2 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Objects; public class DoublesSketchAggregatorFactory extends AggregatorFactory { @@ -73,7 +74,7 @@ public DoublesSketchAggregatorFactory( } this.fieldName = fieldName; this.k = k == null ? DEFAULT_K : k; - Util.checkIfPowerOf2(this.k, "size"); + Util.checkIfPowerOf2(this.k, "k"); this.cacheTypeId = cacheTypeId; } @@ -235,6 +236,9 @@ public boolean equals(final Object o) return false; } final DoublesSketchAggregatorFactory that = (DoublesSketchAggregatorFactory) o; + if (!name.equals(that.name)) { + return false; + } if (!fieldName.equals(that.fieldName)) { return false; } @@ -247,11 +251,7 @@ public boolean equals(final Object o) @Override public int hashCode() { - int result = name.hashCode(); - result = 31 * result + fieldName.hashCode(); - result = 31 * result + Integer.hashCode(k); - result = 31 * result + Byte.hashCode(cacheTypeId); - return result; + return Objects.hash(name, fieldName, k); // no need to use cacheTypeId here } @Override diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java index 54b2edfe2cba..1318c11a725d 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java @@ -113,21 +113,12 @@ public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer o private WritableMemory getMemory(final ByteBuffer buffer) { - WritableMemory mem = memCache.get(buffer); - if (mem == null) { - mem = WritableMemory.wrap(buffer); - memCache.put(buffer, mem); - } - return mem; + return memCache.computeIfAbsent(buffer, buf -> WritableMemory.wrap(buf)); } private void putSketch(final ByteBuffer buffer, final int position, final UpdateDoublesSketch sketch) { - Int2ObjectMap map = sketches.get(buffer); - if (map == null) { - map = new Int2ObjectOpenHashMap<>(); - sketches.put(buffer, map); - } + Int2ObjectMap map = sketches.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>()); map.put(position, sketch); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java index a23fdb822b05..fa35700c6030 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java @@ -58,6 +58,8 @@ public ComplexMetricExtractor getExtractor() { return new ComplexMetricExtractor() { + private static final int MIN_K = 2; // package one input value into the smallest sketch + @Override public Class extractedClass() { @@ -76,7 +78,7 @@ public Object extractValue(final InputRow inputRow, final String metricName) if (Character.isDigit((objectString).charAt(0))) { try { Double doubleValue = Double.parseDouble(objectString); - UpdateDoublesSketch sketch = DoublesSketch.builder().setK(2).build(); + UpdateDoublesSketch sketch = DoublesSketch.builder().setK(MIN_K).build(); sketch.update(doubleValue); return sketch; } @@ -85,7 +87,7 @@ public Object extractValue(final InputRow inputRow, final String metricName) } } } else if (object instanceof Number) { // this is for reindexing - UpdateDoublesSketch sketch = DoublesSketch.builder().setK(2).build(); + UpdateDoublesSketch sketch = DoublesSketch.builder().setK(MIN_K).build(); sketch.update(((Number) object).doubleValue()); return sketch; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java index dbee8d91cc99..b2b51640a833 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java @@ -22,7 +22,6 @@ import java.io.IOException; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; import com.yahoo.sketches.quantiles.DoublesSketch; @@ -32,7 +31,7 @@ public class DoublesSketchJsonSerializer extends JsonSerializer @Override public void serialize(final DoublesSketch sketch, final JsonGenerator generator, final SerializerProvider provider) - throws IOException, JsonProcessingException + throws IOException { generator.writeBinary(sketch.toByteArray(true)); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java index 3a7d5e842c43..8578e7f125cb 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java @@ -119,21 +119,12 @@ public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer o private WritableMemory getMemory(final ByteBuffer buffer) { - WritableMemory mem = memCache.get(buffer); - if (mem == null) { - mem = WritableMemory.wrap(buffer); - memCache.put(buffer, mem); - } - return mem; + return memCache.computeIfAbsent(buffer, buf -> WritableMemory.wrap(buf)); } private void putUnion(final ByteBuffer buffer, final int position, final DoublesUnion union) { - Int2ObjectMap map = unions.get(buffer); - if (map == null) { - map = new Int2ObjectOpenHashMap<>(); - unions.put(buffer, map); - } + Int2ObjectMap map = unions.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>()); map.put(position, union); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java index 706dd1d33622..5a0f70e6b76a 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java @@ -25,6 +25,8 @@ import com.yahoo.memory.Memory; import com.yahoo.sketches.quantiles.DoublesSketch; +import io.druid.java.util.common.ISE; + public class DoublesSketchOperations { @@ -39,7 +41,7 @@ public static DoublesSketch deserialize(final Object serializedSketch) } else if (serializedSketch instanceof DoublesSketch) { return (DoublesSketch) serializedSketch; } - throw new IllegalStateException( + throw new ISE( "Object is not of a type that can be deserialized to a quantiles DoublsSketch: " + serializedSketch.getClass()); } diff --git a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java index eed492c9a9f9..f5c11a056555 100644 --- a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java +++ b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java @@ -30,6 +30,7 @@ import com.yahoo.sketches.quantiles.UpdateDoublesSketch; +// This is used for generating test data for DoublesSketchAggregatorTest public class GenerateTestData { From a5035423e2e71c6ab494a65a49ee72ee66736cdc Mon Sep 17 00:00:00 2001 From: saydakov Date: Mon, 20 Nov 2017 16:51:14 -0800 Subject: [PATCH 14/17] style fix --- .../datasketches/quantiles/DoublesSketchAggregatorFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index e482e54f0aa2..3803c2d7fa0a 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -189,7 +189,8 @@ public List getRequiredColumns() new DoublesSketchAggregatorFactory( fieldName, fieldName, - k)); + k) + ); } @Override From d217263749b6c63589aa6dcdc8f19a7a1417e683 Mon Sep 17 00:00:00 2001 From: saydakov Date: Tue, 21 Nov 2017 11:24:20 -0800 Subject: [PATCH 15/17] use max(k, other.k) for better accuracy --- .../quantiles/DoublesSketchAggregatorFactory.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index 3803c2d7fa0a..93c5cb8c7eca 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -203,7 +203,10 @@ public AggregatorFactory getCombiningFactory() public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException { if (other.getName().equals(this.getName()) && other instanceof DoublesSketchAggregatorFactory) { - return new DoublesSketchMergeAggregatorFactory(name, k); + // DoublesUnion supports inputs with different k. + // The result will have effective k between the specified k and the minimum k from all input sketches + // to achieve higher accuracy as much as possible. + return new DoublesSketchMergeAggregatorFactory(name, Math.max(k, ((DoublesSketchAggregatorFactory) other).k)); } else { throw new AggregatorFactoryNotMergeableException(this, other); } From 3466babc80a3298164d44c25126e3dc15f94d964 Mon Sep 17 00:00:00 2001 From: saydakov Date: Tue, 28 Nov 2017 11:44:27 -0800 Subject: [PATCH 16/17] check for NilColumnValueSelector instead of null --- .../DoublesSketchAggregatorFactory.java | 25 ++++++++----------- .../DoublesSketchMergeAggregatorFactory.java | 5 ++-- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index 93c5cb8c7eca..7ebb40b5d7a5 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -34,6 +34,7 @@ import io.druid.query.cache.CacheKeyBuilder; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; +import io.druid.segment.NilColumnValueSelector; import io.druid.segment.column.ValueType; import java.util.Collections; @@ -83,14 +84,14 @@ public Aggregator factorize(final ColumnSelectorFactory metricFactory) { if (metricFactory.getColumnCapabilities(fieldName) != null && ValueType.isNumeric(metricFactory.getColumnCapabilities(fieldName).getType())) { - final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector == null) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return new DoublesSketchNoOpAggregator(); } - return new DoublesSketchBuildAggregator(valueSelector, k); + return new DoublesSketchBuildAggregator(selector, k); } final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); - if (selector == null) { + if (selector instanceof NilColumnValueSelector) { return new DoublesSketchNoOpAggregator(); } return new DoublesSketchMergeAggregator(selector, k); @@ -101,23 +102,19 @@ public BufferAggregator factorizeBuffered(final ColumnSelectorFactory metricFact { if (metricFactory.getColumnCapabilities(fieldName) != null && ValueType.isNumeric(metricFactory.getColumnCapabilities(fieldName).getType())) { - return getDoubleBufferAggregator(metricFactory.makeColumnValueSelector(fieldName)); + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { + return new DoublesSketchNoOpBufferAggregator(); + } + return new DoublesSketchBuildBufferAggregator(selector, k, getMaxIntermediateSize()); } final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); - if (selector == null) { + if (selector instanceof NilColumnValueSelector) { return new DoublesSketchNoOpBufferAggregator(); } return new DoublesSketchMergeBufferAggregator(selector, k, getMaxIntermediateSize()); } - private BufferAggregator getDoubleBufferAggregator(final ColumnValueSelector selector) - { - if (selector == null) { - return new DoublesSketchNoOpBufferAggregator(); - } - return new DoublesSketchBuildBufferAggregator(selector, k, getMaxIntermediateSize()); - } - @Override public Object deserialize(final Object object) { diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java index 110e8a8523e2..5c3d7b61dcbb 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java @@ -27,6 +27,7 @@ import io.druid.query.aggregation.BufferAggregator; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; +import io.druid.segment.NilColumnValueSelector; public class DoublesSketchMergeAggregatorFactory extends DoublesSketchAggregatorFactory { @@ -43,7 +44,7 @@ public DoublesSketchMergeAggregatorFactory( public Aggregator factorize(final ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); - if (selector == null) { + if (selector instanceof NilColumnValueSelector) { return new DoublesSketchNoOpAggregator(); } return new DoublesSketchMergeAggregator(selector, getK()); @@ -53,7 +54,7 @@ public Aggregator factorize(final ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(final ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); - if (selector == null) { + if (selector instanceof NilColumnValueSelector) { return new DoublesSketchNoOpBufferAggregator(); } return new DoublesSketchMergeBufferAggregator(selector, getK(), getMaxIntermediateSize()); From 6d2f90338179543efcea374a4a132995ffa2830f Mon Sep 17 00:00:00 2001 From: saydakov Date: Tue, 28 Nov 2017 12:57:22 -0800 Subject: [PATCH 17/17] throw exceptions instead of providing no-op comparators --- .../DoublesSketchToHistogramPostAggregator.java | 13 +++---------- .../DoublesSketchToQuantilePostAggregator.java | 4 ++-- .../DoublesSketchToQuantilesPostAggregator.java | 14 +++----------- .../DoublesSketchToStringPostAggregator.java | 14 +++----------- 4 files changed, 11 insertions(+), 34 deletions(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java index b3f862f0da7d..015d7787d43c 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.java.util.common.IAE; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; @@ -83,18 +84,10 @@ public double[] getSplitPoints() return splitPoints; } - // comparing histograms doesn't make much sense, so this comparator pretends that everything is equal @Override public Comparator getComparator() { - return new Comparator() - { - @Override - public int compare(double[] o1, double[] o2) - { - return 0; - } - }; + throw new IAE("Comparing histograms is not supported"); } @Override @@ -152,7 +145,7 @@ public byte[] getCacheKey() } @Override - public PostAggregator decorate(Map map) + public PostAggregator decorate(final Map map) { return this; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java index 9aacc8bb005c..a9df26b57743 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java @@ -85,7 +85,7 @@ public Comparator getComparator() return new Comparator() { @Override - public int compare(Double a, Double b) + public int compare(final Double a, final Double b) { return Doubles.compare(a, b); } @@ -141,7 +141,7 @@ public byte[] getCacheKey() } @Override - public PostAggregator decorate(Map map) + public PostAggregator decorate(final Map map) { return this; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java index 2da179ecba4c..45674dcf7f71 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.java.util.common.IAE; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; @@ -79,19 +80,10 @@ public Object compute(final Map combinedAggregators) return sketch.getQuantiles(fractions); } - // comparing arrays of quantiles doesn't make much sense, so this comparator - // pretends everything is equal @Override public Comparator getComparator() { - return new Comparator() - { - @Override - public int compare(double[] o1, double[] o2) - { - return 0; - } - }; + throw new IAE("Comparing arrays of quantiles is not supported"); } @Override @@ -147,7 +139,7 @@ public byte[] getCacheKey() } @Override - public PostAggregator decorate(Map map) + public PostAggregator decorate(final Map map) { return this; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java index ceeb7d41ece1..204b80cef50c 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java @@ -27,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.java.util.common.IAE; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; @@ -69,19 +70,10 @@ public Object compute(final Map combinedAggregators) return sketch.toString(); } - // comparing sketch summaries doesn't make much sense, so this comparator - // pretends everything is equal @Override public Comparator getComparator() { - return new Comparator() - { - @Override - public int compare(String o1, String o2) - { - return 0; - } - }; + throw new IAE("Comparing sketch summaries is not supported"); } @Override @@ -130,7 +122,7 @@ public byte[] getCacheKey() } @Override - public PostAggregator decorate(Map map) + public PostAggregator decorate(final Map map) { return this; }