From 6664c3d13afdf886e818ad70dcbfaecc89bdd1a1 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Tue, 15 Nov 2022 10:19:18 -0800 Subject: [PATCH 01/10] Unsigned integer druid complex column --- .../extensions-contrib/unsigned-int.md | 47 ++++++++ docs/development/extensions.md | 1 + extensions-contrib/unsigned-int/pom.xml | 74 ++++++++++++ .../druid/uint/UnSignedIntObjectStrategy.java | 68 +++++++++++ .../druid/uint/UnsignedIntComplexSerde.java | 77 ++++++++++++ .../druid/uint/UnsignedIntDruidModule.java | 58 +++++++++ .../aggs/UnsignedIntAnyAggregatorFactory.java | 51 ++++++++ .../aggs/UnsignedIntMaxAggregatorFactory.java | 62 ++++++++++ .../aggs/UnsignedIntMinAggregatorFactory.java | 62 ++++++++++ .../aggs/UnsignedIntSumAggregatorFactory.java | 63 ++++++++++ ...rg.apache.druid.initialization.DruidModule | 1 + .../org/apache/druid/uint/DruidBaseTest.java | 91 +++++++++++++++ .../uint/UnSignedIntObjectStrategyTest.java | 50 ++++++++ .../uint/UnsignedIntAggregationTest.java | 110 ++++++++++++++++++ .../uint/UnsignedIntDruidModuleTest.java | 41 +++++++ .../src/test/resources/index_agg.json | 7 ++ .../resources/queries/group_by_max_query.json | 18 +++ .../resources/queries/group_by_min_query.json | 18 +++ .../resources/queries/group_by_sum_query.json | 18 +++ .../unsigned-int/src/test/resources/spec.json | 17 +++ .../src/test/resources/u_int_data.dat | 6 + pom.xml | 1 + 22 files changed, 941 insertions(+) create mode 100644 docs/development/extensions-contrib/unsigned-int.md create mode 100644 extensions-contrib/unsigned-int/pom.xml create mode 100644 extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java create mode 100644 extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java create mode 100644 extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java create mode 100644 extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java create mode 100644 extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java create mode 100644 extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java create mode 100644 extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java create mode 100644 extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java create mode 100644 extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnSignedIntObjectStrategyTest.java create mode 100644 extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java create mode 100644 extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntDruidModuleTest.java create mode 100644 extensions-contrib/unsigned-int/src/test/resources/index_agg.json create mode 100644 extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json create mode 100644 extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json create mode 100644 extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json create mode 100644 extensions-contrib/unsigned-int/src/test/resources/spec.json create mode 100644 extensions-contrib/unsigned-int/src/test/resources/u_int_data.dat diff --git a/docs/development/extensions-contrib/unsigned-int.md b/docs/development/extensions-contrib/unsigned-int.md new file mode 100644 index 000000000000..083a7390e5c6 --- /dev/null +++ b/docs/development/extensions-contrib/unsigned-int.md @@ -0,0 +1,47 @@ +--- +id: unsigned-int +title: "Un-signed Integer Column Type" +--- + + + +Apache Druid Extension to utilize an un-signed int column. + +Consider this an [EXPERIMENTAL](../experimental.md) feature mostly because it has not been tested yet on a wide variety of long-running Druid clusters. + +## How it works + +Only store a 4 byte un-signed integer instead of a long column when you need your columns don't require you to store a long. Aggregations still happen as long values, only the data stored on disk is as an un-signed integer. + +## Configuration + +To use this extension please make sure to [include](../extensions.md#loading-extensions)`unsigned-int-extensions` in the extensions load list. + +Example Usage: + +``` +"metricsSpec": [ + { + "type": "unsigned_int", + "name": "value", + "fieldName": "value" + }, +``` + diff --git a/docs/development/extensions.md b/docs/development/extensions.md index 36d3549b195e..7571425a03d6 100644 --- a/docs/development/extensions.md +++ b/docs/development/extensions.md @@ -98,6 +98,7 @@ All of these community extensions can be downloaded using [pull-deps](../operati |gce-extensions|GCE Extensions|[link](../development/extensions-contrib/gce-extensions.md)| |prometheus-emitter|Exposes [Druid metrics](../operations/metrics.md) for Prometheus server collection (https://prometheus.io/)|[link](./extensions-contrib/prometheus.md)| |kubernetes-overlord-extensions|Support for launching tasks in k8s without Middle Managers|[link](../development/extensions-contrib/k8s-jobs.md)| +|unsigned-int-extensions|Support for an un-signed integer column|[link](../development/extensions-contrib/unsigned-int.md)| ## Promoting community extensions to core extensions diff --git a/extensions-contrib/unsigned-int/pom.xml b/extensions-contrib/unsigned-int/pom.xml new file mode 100644 index 000000000000..4f3eeecdeb16 --- /dev/null +++ b/extensions-contrib/unsigned-int/pom.xml @@ -0,0 +1,74 @@ + + + + + 4.0.0 + + org.apache.druid.extensions.contrib + unsigned-int-extensions + unsigned-int-extensions + unsigned-int-extensions + + + org.apache.druid + druid + 25.0.0-SNAPSHOT + ../../pom.xml + + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + org.apache.druid + druid-processing + ${project.parent.version} + provided + + + + org.apache.druid + druid-processing + ${project.parent.version} + test-jar + test + + + org.apache.druid + druid-core + ${project.parent.version} + test-jar + test + + + junit + junit + test + + + + + diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java new file mode 100644 index 000000000000..13eb2766c027 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import org.apache.druid.segment.data.ObjectStrategy; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class UnSignedIntObjectStrategy implements ObjectStrategy +{ + @Override + public Class getClazz() + { + return Long.class; + } + + @Nullable + @Override + public Long fromByteBuffer(ByteBuffer buffer, int numBytes) + { + if (numBytes <= 0) { + return null; + } + byte[] cache = new byte[numBytes]; + buffer.get(cache); + ByteBuffer result = ByteBuffer.allocate(8).put(new byte[]{0, 0, 0, 0}).put(cache); + result.position(0); + return result.getLong(); + + } + + @Nullable + @Override + public byte[] toBytes(@Nullable Long val) + { + if (val == null) { + return new byte[0]; + } + byte[] bytes = new byte[8]; + ByteBuffer.wrap(bytes).putLong(val); + return Arrays.copyOfRange(bytes, 4, 8); + } + + @Override + public int compare(Long left, Long right) + { + return left.compareTo(right); + } +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java new file mode 100644 index 000000000000..5493cfc3ed96 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.ComplexColumnPartSupplier; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.apache.druid.segment.serde.ComplexMetricSerde; + +import java.nio.ByteBuffer; + +public class UnsignedIntComplexSerde extends ComplexMetricSerde +{ + + public static final String TYPE = "unsigned_int"; + private final UnSignedIntObjectStrategy strategy = new UnSignedIntObjectStrategy(); + + + @Override + public String getTypeName() + { + return TYPE; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + return new ComplexMetricExtractor() + { + @Override + public Class extractedClass() + { + return Long.class; + } + + @Override + public Long extractValue(InputRow inputRow, String metricName) + { + Object obj = inputRow.getRaw(metricName); + return (Long) obj; + } + }; + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder) + { + GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), builder.getFileMapper()); + builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return strategy; + } +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java new file mode 100644 index 000000000000..6251ec020c61 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.segment.serde.ComplexMetrics; +import org.apache.druid.uint.aggs.UnsignedIntAnyAggregatorFactory; +import org.apache.druid.uint.aggs.UnsignedIntMaxAggregatorFactory; +import org.apache.druid.uint.aggs.UnsignedIntMinAggregatorFactory; +import org.apache.druid.uint.aggs.UnsignedIntSumAggregatorFactory; + +import java.util.Arrays; +import java.util.List; + +public class UnsignedIntDruidModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return Arrays.asList(new SimpleModule("UnsignedIntDruidModule") + .registerSubtypes( + new NamedType(UnsignedIntSumAggregatorFactory.class, UnsignedIntComplexSerde.TYPE), + new NamedType(UnsignedIntMaxAggregatorFactory.class, UnsignedIntComplexSerde.TYPE), + new NamedType(UnsignedIntMinAggregatorFactory.class, UnsignedIntComplexSerde.TYPE), + new NamedType(UnsignedIntAnyAggregatorFactory.class, UnsignedIntComplexSerde.TYPE) + ) + ); + } + + @Override + public void configure(Binder binder) + { + if (ComplexMetrics.getSerdeForType(UnsignedIntComplexSerde.TYPE) == null) { + ComplexMetrics.registerSerde(UnsignedIntComplexSerde.TYPE, new UnsignedIntComplexSerde()); + } + } +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java new file mode 100644 index 000000000000..8e1196e175c5 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint.aggs; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.uint.UnsignedIntComplexSerde; + +public class UnsignedIntAnyAggregatorFactory extends LongAnyAggregatorFactory +{ + + @JsonCreator + public UnsignedIntAnyAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") final String fieldName + ) + { + super(name, fieldName); + } + + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + + @Override + public String getComplexTypeName() + { + return UnsignedIntComplexSerde.TYPE; + } +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java new file mode 100644 index 000000000000..8842fc6671b4 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint.aggs; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.uint.UnsignedIntComplexSerde; + +import javax.annotation.Nullable; + +public class UnsignedIntMaxAggregatorFactory extends LongMaxAggregatorFactory +{ + + @JsonCreator + public UnsignedIntMaxAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("expression") @Nullable String expression, + @JacksonInject ExprMacroTable macroTable + ) + { + super(name, fieldName, expression, macroTable); + } + + public UnsignedIntMaxAggregatorFactory(String name, String fieldName) + { + super(name, fieldName); + } + + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + + @Override + public String getComplexTypeName() + { + return UnsignedIntComplexSerde.TYPE; + } +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java new file mode 100644 index 000000000000..2ed2c55e28bf --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint.aggs; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.LongMinAggregatorFactory; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.uint.UnsignedIntComplexSerde; + +import javax.annotation.Nullable; + +public class UnsignedIntMinAggregatorFactory extends LongMinAggregatorFactory +{ + + @JsonCreator + public UnsignedIntMinAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("expression") @Nullable String expression, + @JacksonInject ExprMacroTable macroTable + ) + { + super(name, fieldName, expression, macroTable); + } + + public UnsignedIntMinAggregatorFactory(String name, String fieldName) + { + super(name, fieldName); + } + + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + + @Override + public String getComplexTypeName() + { + return UnsignedIntComplexSerde.TYPE; + } +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java new file mode 100644 index 000000000000..f3a67512012a --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint.aggs; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.uint.UnsignedIntComplexSerde; + +import javax.annotation.Nullable; + +public class UnsignedIntSumAggregatorFactory extends LongSumAggregatorFactory +{ + + @JsonCreator + public UnsignedIntSumAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("expression") @Nullable String expression, + @JacksonInject ExprMacroTable macroTable + ) + { + super(name, fieldName, expression, macroTable); + } + + + public UnsignedIntSumAggregatorFactory(String name, String fieldName) + { + super(name, fieldName); + } + + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + + @Override + public String getComplexTypeName() + { + return UnsignedIntComplexSerde.TYPE; + } +} diff --git a/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..8c56e227b8ac --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1 @@ +org.apache.druid.uint.UnsignedIntDruidModule \ No newline at end of file diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java new file mode 100644 index 000000000000..28da0ae6437a --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.aggregation.AggregationTestHelper; +import org.apache.druid.query.groupby.GroupByQueryConfig; +import org.apache.druid.query.groupby.strategy.GroupByStrategySelector; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Objects; + +public abstract class DruidBaseTest +{ + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + protected final AggregationTestHelper groupByTestHelper; + protected final ObjectMapper jsonMapper; + + public DruidBaseTest() + { + UnsignedIntDruidModule module = new UnsignedIntDruidModule(); + module.configure(null); + GroupByQueryConfig config = new GroupByQueryConfig() + { + @Override + public String getDefaultStrategy() + { + return GroupByStrategySelector.STRATEGY_V2; + } + }; + + groupByTestHelper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( + module.getJacksonModules(), config, tempFolder + ); + + jsonMapper = groupByTestHelper.getObjectMapper(); + } + + public static void createSegmentFromDataFile( + AggregationTestHelper aggregationTestHelper, + String dataFile, File segmentDir + ) throws Exception + { + aggregationTestHelper.createIndex( + Paths.get(Objects.requireNonNull(DruidBaseTest.class.getClassLoader().getResource(dataFile)) + .toURI()).toFile(), + readJsonAsString("spec.json"), + readJsonAsString("index_agg.json"), + segmentDir, + 0, + Granularities.NONE, + 2, + false + ); + } + + public static String readJsonAsString(String s) throws IOException, URISyntaxException + { + return String.join(" ", Files.readAllLines(Paths + .get(Objects.requireNonNull(DruidBaseTest.class.getClassLoader() + .getResource(s)) + .toURI()))); + } + +} diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnSignedIntObjectStrategyTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnSignedIntObjectStrategyTest.java new file mode 100644 index 000000000000..d72fe4cfc53b --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnSignedIntObjectStrategyTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class UnSignedIntObjectStrategyTest +{ + + @Test + public void testSimpleCase() + { + UnSignedIntObjectStrategy strategy = new UnSignedIntObjectStrategy(); + byte[] bytes = strategy.toBytes(2L); + assertEquals(4, bytes.length); + Long result = strategy.fromByteBuffer(ByteBuffer.wrap(bytes), bytes.length); + assertEquals(Long.valueOf(2), result); + } + + @Test + public void testNullObject() + { + UnSignedIntObjectStrategy strategy = new UnSignedIntObjectStrategy(); + byte[] bytes = strategy.toBytes(null); + Long result = strategy.fromByteBuffer(ByteBuffer.wrap(bytes), bytes.length); + assertNull(result); + } +} diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java new file mode 100644 index 000000000000..65b50a45ac14 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.query.groupby.ResultRow; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class UnsignedIntAggregationTest extends DruidBaseTest +{ + + private File segmentDir; + + @Before + public void setUp() throws Exception + { + segmentDir = tempFolder.newFolder(); + } + + @After + public void tearDown() throws Exception + { + segmentDir.delete(); + } + + @Test + public void testSumAgg() throws Exception + { + createSegmentFromDataFile(groupByTestHelper, "u_int_data.dat", segmentDir); + Sequence sequence = groupByTestHelper.runQueryOnSegments( + ImmutableList.of(segmentDir), + readJsonAsString("queries/group_by_sum_query.json") + ); + + List actual = sequence.toList(); + List expected = Lists.newArrayList( + ResultRow.of("a", 199L), + ResultRow.of("b", 98L), + ResultRow.of("c", (long) 2 * Integer.MAX_VALUE) // if only int agg, it would overflow. + ); + assertEquals(3, actual.size()); + assertArrayEquals(expected.toArray(), actual.toArray()); + } + + @Test + public void testMaxAgg() throws Exception + { + createSegmentFromDataFile(groupByTestHelper, "u_int_data.dat", segmentDir); + Sequence sequence = groupByTestHelper.runQueryOnSegments( + ImmutableList.of(segmentDir), + readJsonAsString("queries/group_by_max_query.json") + ); + + List actual = sequence.toList(); + List expected = Lists.newArrayList( + ResultRow.of("a", 100L), + ResultRow.of("b", 97L), + ResultRow.of("c", (long) Integer.MAX_VALUE) // if only int agg, it would overflow. + ); + assertEquals(3, actual.size()); + assertArrayEquals(expected.toArray(), actual.toArray()); + } + + @Test + public void testMinAgg() throws Exception + { + createSegmentFromDataFile(groupByTestHelper, "u_int_data.dat", segmentDir); + Sequence sequence = groupByTestHelper.runQueryOnSegments( + ImmutableList.of(segmentDir), + readJsonAsString("queries/group_by_min_query.json") + ); + + List actual = sequence.toList(); + List expected = Lists.newArrayList( + ResultRow.of("a", 99L), + ResultRow.of("b", 1L), + ResultRow.of("c", (long) Integer.MAX_VALUE) // if only int agg, it would overflow. + ); + assertEquals(3, actual.size()); + assertArrayEquals(expected.toArray(), actual.toArray()); + } + +} diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntDruidModuleTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntDruidModuleTest.java new file mode 100644 index 000000000000..bdd1aaa9291f --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntDruidModuleTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.fasterxml.jackson.databind.Module; +import com.google.common.collect.Iterables; +import org.junit.Test; + +import java.util.List; + +import static junit.framework.Assert.assertEquals; + +public class UnsignedIntDruidModuleTest +{ + + @Test + public void testSomethingSimple() + { + List result = new UnsignedIntDruidModule().getJacksonModules(); + assertEquals(1, result.size()); + assertEquals("UnsignedIntDruidModule", Iterables.getOnlyElement(result).getModuleName()); + } + +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/index_agg.json b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json new file mode 100644 index 000000000000..705d69eafaa9 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json @@ -0,0 +1,7 @@ +[ + { + "name": "u_int", + "type": "unsigned_int", + "fieldName": "foo" + } +] diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json new file mode 100644 index 000000000000..d097a2351d5b --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json @@ -0,0 +1,18 @@ +{ + "queryType": "groupBy", + "dataSource": "test_datasource", + "granularity": "ALL", + "aggregations": [ + { + "type": "longMax", + "name": "out", + "fieldName": "foo" + } + ], + "dimensions": [ + "dim" + ], + "intervals": [ + "1970-01-01T00:00:00.000Z/2020-10-22T00:00:00.000Z" + ] +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json new file mode 100644 index 000000000000..8c8aaa17e9a1 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json @@ -0,0 +1,18 @@ +{ + "queryType": "groupBy", + "dataSource": "test_datasource", + "granularity": "ALL", + "aggregations": [ + { + "type": "longMin", + "name": "out", + "fieldName": "foo" + } + ], + "dimensions": [ + "dim" + ], + "intervals": [ + "1970-01-01T00:00:00.000Z/2020-10-22T00:00:00.000Z" + ] +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json new file mode 100644 index 000000000000..d0e45995cc86 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json @@ -0,0 +1,18 @@ +{ + "queryType": "groupBy", + "dataSource": "test_datasource", + "granularity": "ALL", + "aggregations": [ + { + "type": "longSum", + "name": "out", + "fieldName": "foo" + } + ], + "dimensions": [ + "dim" + ], + "intervals": [ + "1970-01-01T00:00:00.000Z/2020-10-22T00:00:00.000Z" + ] +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/spec.json b/extensions-contrib/unsigned-int/src/test/resources/spec.json new file mode 100644 index 000000000000..51f72d722a6a --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/spec.json @@ -0,0 +1,17 @@ +{ + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "timestamp" + ], + "spatialDimensions": [] + } + } +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/u_int_data.dat b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.dat new file mode 100644 index 000000000000..6d26d1f981f5 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.dat @@ -0,0 +1,6 @@ +{"timestamp": 100, "dim": "a", "foo": 100} +{"timestamp": 101, "dim": "a", "foo": 99} +{"timestamp": 102, "dim": "b", "foo": 97} +{"timestamp": 104, "dim": "b", "foo": 1} +{"timestamp": 104, "dim": "c", "foo": 2147483647} +{"timestamp": 105, "dim": "c", "foo": 2147483647} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5e44487413d2..da3fe1b7e5ed 100644 --- a/pom.xml +++ b/pom.xml @@ -215,6 +215,7 @@ extensions-contrib/prometheus-emitter extensions-contrib/opentelemetry-emitter extensions-contrib/kubernetes-overlord-extensions + extensions-contrib/unsigned-int distribution From 5e99bfc3a21dd6195d5e32fbc3d70e6194f468a9 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Tue, 15 Nov 2022 13:55:00 -0800 Subject: [PATCH 02/10] Travis CI Fixes --- extensions-contrib/unsigned-int/pom.xml | 26 +++++++++++- .../druid/uint/UnsignedIntDruidModule.java | 29 +++++++++---- .../aggs/UnsignedIntAnyAggregatorFactory.java | 13 +++--- .../aggs/UnsignedIntMaxAggregatorFactory.java | 18 +++----- .../aggs/UnsignedIntMinAggregatorFactory.java | 19 +++------ .../aggs/UnsignedIntSumAggregatorFactory.java | 19 +++------ .../uint/UnsignedIntAggregationTest.java | 2 +- .../uint/UnsignedIntComplexSerdeTest.java | 42 +++++++++++++++++++ 8 files changed, 109 insertions(+), 59 deletions(-) create mode 100644 extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntComplexSerdeTest.java diff --git a/extensions-contrib/unsigned-int/pom.xml b/extensions-contrib/unsigned-int/pom.xml index 4f3eeecdeb16..9cc9e4d6493b 100644 --- a/extensions-contrib/unsigned-int/pom.xml +++ b/extensions-contrib/unsigned-int/pom.xml @@ -48,7 +48,31 @@ ${project.parent.version} provided - + + com.google.code.findbugs + jsr305 + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.google.guava + guava + provided + + + com.google.inject + guice + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + org.apache.druid druid-processing diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java index 6251ec020c61..753ebf8147b7 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java @@ -30,7 +30,7 @@ import org.apache.druid.uint.aggs.UnsignedIntMinAggregatorFactory; import org.apache.druid.uint.aggs.UnsignedIntSumAggregatorFactory; -import java.util.Arrays; +import java.util.Collections; import java.util.List; public class UnsignedIntDruidModule implements DruidModule @@ -38,13 +38,26 @@ public class UnsignedIntDruidModule implements DruidModule @Override public List getJacksonModules() { - return Arrays.asList(new SimpleModule("UnsignedIntDruidModule") - .registerSubtypes( - new NamedType(UnsignedIntSumAggregatorFactory.class, UnsignedIntComplexSerde.TYPE), - new NamedType(UnsignedIntMaxAggregatorFactory.class, UnsignedIntComplexSerde.TYPE), - new NamedType(UnsignedIntMinAggregatorFactory.class, UnsignedIntComplexSerde.TYPE), - new NamedType(UnsignedIntAnyAggregatorFactory.class, UnsignedIntComplexSerde.TYPE) - ) + return Collections.singletonList( + new SimpleModule("UnsignedIntDruidModule") + .registerSubtypes( + new NamedType( + UnsignedIntSumAggregatorFactory.class, + UnsignedIntComplexSerde.TYPE + ), + new NamedType( + UnsignedIntMaxAggregatorFactory.class, + UnsignedIntComplexSerde.TYPE + ), + new NamedType( + UnsignedIntMinAggregatorFactory.class, + UnsignedIntComplexSerde.TYPE + ), + new NamedType( + UnsignedIntAnyAggregatorFactory.class, + UnsignedIntComplexSerde.TYPE + ) + ) ); } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java index 8e1196e175c5..e4c07993750c 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java @@ -22,12 +22,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory; -import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.uint.UnsignedIntComplexSerde; public class UnsignedIntAnyAggregatorFactory extends LongAnyAggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(UnsignedIntComplexSerde.TYPE); + @JsonCreator public UnsignedIntAnyAggregatorFactory( @JsonProperty("name") String name, @@ -38,14 +40,9 @@ public UnsignedIntAnyAggregatorFactory( } @Override - public ValueType getType() + public ColumnType getIntermediateType() { - return ValueType.COMPLEX; + return TYPE; } - @Override - public String getComplexTypeName() - { - return UnsignedIntComplexSerde.TYPE; - } } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java index 8842fc6671b4..cabba5fae280 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; -import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.uint.UnsignedIntComplexSerde; import javax.annotation.Nullable; @@ -32,6 +32,8 @@ public class UnsignedIntMaxAggregatorFactory extends LongMaxAggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(UnsignedIntComplexSerde.TYPE); + @JsonCreator public UnsignedIntMaxAggregatorFactory( @JsonProperty("name") String name, @@ -43,20 +45,10 @@ public UnsignedIntMaxAggregatorFactory( super(name, fieldName, expression, macroTable); } - public UnsignedIntMaxAggregatorFactory(String name, String fieldName) - { - super(name, fieldName); - } - @Override - public ValueType getType() + public ColumnType getIntermediateType() { - return ValueType.COMPLEX; + return TYPE; } - @Override - public String getComplexTypeName() - { - return UnsignedIntComplexSerde.TYPE; - } } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java index 2ed2c55e28bf..9a4b5bccc11b 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.aggregation.LongMinAggregatorFactory; -import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.uint.UnsignedIntComplexSerde; import javax.annotation.Nullable; @@ -32,6 +32,8 @@ public class UnsignedIntMinAggregatorFactory extends LongMinAggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(UnsignedIntComplexSerde.TYPE); + @JsonCreator public UnsignedIntMinAggregatorFactory( @JsonProperty("name") String name, @@ -43,20 +45,9 @@ public UnsignedIntMinAggregatorFactory( super(name, fieldName, expression, macroTable); } - public UnsignedIntMinAggregatorFactory(String name, String fieldName) - { - super(name, fieldName); - } - - @Override - public ValueType getType() - { - return ValueType.COMPLEX; - } - @Override - public String getComplexTypeName() + public ColumnType getIntermediateType() { - return UnsignedIntComplexSerde.TYPE; + return TYPE; } } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java index f3a67512012a..7ec1d0ada1bc 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.uint.UnsignedIntComplexSerde; import javax.annotation.Nullable; @@ -32,6 +32,8 @@ public class UnsignedIntSumAggregatorFactory extends LongSumAggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(UnsignedIntComplexSerde.TYPE); + @JsonCreator public UnsignedIntSumAggregatorFactory( @JsonProperty("name") String name, @@ -43,21 +45,10 @@ public UnsignedIntSumAggregatorFactory( super(name, fieldName, expression, macroTable); } - - public UnsignedIntSumAggregatorFactory(String name, String fieldName) - { - super(name, fieldName); - } - @Override - public ValueType getType() + public ColumnType getIntermediateType() { - return ValueType.COMPLEX; + return TYPE; } - @Override - public String getComplexTypeName() - { - return UnsignedIntComplexSerde.TYPE; - } } diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java index 65b50a45ac14..c40cdb79838a 100644 --- a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java @@ -45,7 +45,7 @@ public void setUp() throws Exception } @After - public void tearDown() throws Exception + public void tearDown() { segmentDir.delete(); } diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntComplexSerdeTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntComplexSerdeTest.java new file mode 100644 index 000000000000..babad205e919 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntComplexSerdeTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.junit.Assert; +import org.junit.Test; + +public class UnsignedIntComplexSerdeTest +{ + @Test + public void testSerde() + { + final UnsignedIntComplexSerde serde = new UnsignedIntComplexSerde(); + final ComplexMetricExtractor extractor = serde.getExtractor(); + final Long value = (Long) extractor.extractValue( + new MapBasedInputRow(0L, ImmutableList.of(), ImmutableMap.of("foo", 3L)), + "foo" + ); + Assert.assertEquals(Long.valueOf(3), value); + } +} From e8ad174cb6be827937c967025c0376d9586d89c8 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Wed, 16 Nov 2022 10:11:05 -0800 Subject: [PATCH 03/10] Needed header for initialization file --- .../org.apache.druid.initialization.DruidModule | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index 8c56e227b8ac..197142de0469 100644 --- a/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -1 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + org.apache.druid.uint.UnsignedIntDruidModule \ No newline at end of file From 5d9d6e7ab89e8e51e809db0ed24fcf5abc81a7fb Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Wed, 16 Nov 2022 11:50:48 -0800 Subject: [PATCH 04/10] Fixing the spelling for the docs --- docs/development/extensions-contrib/unsigned-int.md | 6 +++--- docs/development/extensions.md | 2 +- website/.spelling | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/development/extensions-contrib/unsigned-int.md b/docs/development/extensions-contrib/unsigned-int.md index 083a7390e5c6..cbfc5fe03205 100644 --- a/docs/development/extensions-contrib/unsigned-int.md +++ b/docs/development/extensions-contrib/unsigned-int.md @@ -1,6 +1,6 @@ --- id: unsigned-int -title: "Un-signed Integer Column Type" +title: "Unsigned Integer Column Type" --- -Apache Druid Extension to utilize an un-signed int column. +Apache Druid Extension to utilize an unsigned int column. Consider this an [EXPERIMENTAL](../experimental.md) feature mostly because it has not been tested yet on a wide variety of long-running Druid clusters. ## How it works -Only store a 4 byte un-signed integer instead of a long column when you need your columns don't require you to store a long. Aggregations still happen as long values, only the data stored on disk is as an un-signed integer. +Only store a 4 byte unsigned integer instead of a long column when you need your columns don't require you to store a long. Aggregations still happen as long values, only the data stored on disk is as an unsigned integer. ## Configuration diff --git a/docs/development/extensions.md b/docs/development/extensions.md index 7571425a03d6..b165f88f3b75 100644 --- a/docs/development/extensions.md +++ b/docs/development/extensions.md @@ -98,7 +98,7 @@ All of these community extensions can be downloaded using [pull-deps](../operati |gce-extensions|GCE Extensions|[link](../development/extensions-contrib/gce-extensions.md)| |prometheus-emitter|Exposes [Druid metrics](../operations/metrics.md) for Prometheus server collection (https://prometheus.io/)|[link](./extensions-contrib/prometheus.md)| |kubernetes-overlord-extensions|Support for launching tasks in k8s without Middle Managers|[link](../development/extensions-contrib/k8s-jobs.md)| -|unsigned-int-extensions|Support for an un-signed integer column|[link](../development/extensions-contrib/unsigned-int.md)| +|unsigned-int-extensions|Support for an unsigned integer column|[link](../development/extensions-contrib/unsigned-int.md)| ## Promoting community extensions to core extensions diff --git a/website/.spelling b/website/.spelling index d02f4f846799..436468e5c65c 100644 --- a/website/.spelling +++ b/website/.spelling @@ -506,6 +506,7 @@ unnests unparseable unparsed unsetting +unsigned untrusted useFilterCNF useJqSyntax From ee5c4e598b204ac8865f8214f2162719a1152edd Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Wed, 16 Nov 2022 13:14:19 -0800 Subject: [PATCH 05/10] Rat requires license headers for test data, trying to work around this --- .../org/apache/druid/uint/DruidBaseTest.java | 18 ++++++++++++++---- .../druid/uint/UnsignedIntAggregationTest.java | 6 +++--- .../src/test/resources/index_agg.json | 15 +++++++++++++++ .../resources/queries/group_by_max_query.json | 15 +++++++++++++++ .../resources/queries/group_by_min_query.json | 15 +++++++++++++++ .../resources/queries/group_by_sum_query.json | 15 +++++++++++++++ .../unsigned-int/src/test/resources/spec.json | 15 +++++++++++++++ .../{u_int_data.dat => u_int_data.data} | 0 8 files changed, 92 insertions(+), 7 deletions(-) rename extensions-contrib/unsigned-int/src/test/resources/{u_int_data.dat => u_int_data.data} (100%) diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java index 28da0ae6437a..0a51e920e695 100644 --- a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java @@ -32,6 +32,7 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.List; import java.util.Objects; public abstract class DruidBaseTest @@ -82,10 +83,19 @@ public static void createSegmentFromDataFile( public static String readJsonAsString(String s) throws IOException, URISyntaxException { - return String.join(" ", Files.readAllLines(Paths - .get(Objects.requireNonNull(DruidBaseTest.class.getClassLoader() - .getResource(s)) - .toURI()))); + List lines = Files.readAllLines( + Paths + .get(Objects.requireNonNull(DruidBaseTest.class.getClassLoader() + .getResource( + s)) + .toURI())); + StringBuilder builder = new StringBuilder(); + for (String line : lines) { + if (!line.trim().startsWith("//")) { + builder.append(line).append(" "); + } + } + return builder.toString(); } } diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java index c40cdb79838a..ce8b90dd64e6 100644 --- a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java @@ -53,7 +53,7 @@ public void tearDown() @Test public void testSumAgg() throws Exception { - createSegmentFromDataFile(groupByTestHelper, "u_int_data.dat", segmentDir); + createSegmentFromDataFile(groupByTestHelper, "u_int_data.data", segmentDir); Sequence sequence = groupByTestHelper.runQueryOnSegments( ImmutableList.of(segmentDir), readJsonAsString("queries/group_by_sum_query.json") @@ -72,7 +72,7 @@ public void testSumAgg() throws Exception @Test public void testMaxAgg() throws Exception { - createSegmentFromDataFile(groupByTestHelper, "u_int_data.dat", segmentDir); + createSegmentFromDataFile(groupByTestHelper, "u_int_data.data", segmentDir); Sequence sequence = groupByTestHelper.runQueryOnSegments( ImmutableList.of(segmentDir), readJsonAsString("queries/group_by_max_query.json") @@ -91,7 +91,7 @@ public void testMaxAgg() throws Exception @Test public void testMinAgg() throws Exception { - createSegmentFromDataFile(groupByTestHelper, "u_int_data.dat", segmentDir); + createSegmentFromDataFile(groupByTestHelper, "u_int_data.data", segmentDir); Sequence sequence = groupByTestHelper.runQueryOnSegments( ImmutableList.of(segmentDir), readJsonAsString("queries/group_by_min_query.json") diff --git a/extensions-contrib/unsigned-int/src/test/resources/index_agg.json b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json index 705d69eafaa9..339eba7e1988 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/index_agg.json +++ b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + [ { "name": "u_int", diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json index d097a2351d5b..572edf63e1cb 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + { "queryType": "groupBy", "dataSource": "test_datasource", diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json index 8c8aaa17e9a1..499f46c5aba2 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + { "queryType": "groupBy", "dataSource": "test_datasource", diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json index d0e45995cc86..891c83f6f0b7 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + { "queryType": "groupBy", "dataSource": "test_datasource", diff --git a/extensions-contrib/unsigned-int/src/test/resources/spec.json b/extensions-contrib/unsigned-int/src/test/resources/spec.json index 51f72d722a6a..db62bb36e9c9 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/spec.json +++ b/extensions-contrib/unsigned-int/src/test/resources/spec.json @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + { "type": "string", "parseSpec": { diff --git a/extensions-contrib/unsigned-int/src/test/resources/u_int_data.dat b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data similarity index 100% rename from extensions-contrib/unsigned-int/src/test/resources/u_int_data.dat rename to extensions-contrib/unsigned-int/src/test/resources/u_int_data.data From 4fd4d63e982bb8691e509b89053fc8c76c3fb8f9 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Thu, 17 Nov 2022 16:52:27 -0800 Subject: [PATCH 06/10] Things finally work --- .../org/apache/druid/uint/UnSignedIntObjectStrategy.java | 3 ++- .../org/apache/druid/uint/UnsignedIntDruidModule.java | 8 ++++---- .../druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java | 5 +++++ .../druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java | 6 ++++++ .../druid/uint/aggs/UnsignedIntMinAggregatorFactory.java | 6 ++++++ .../druid/uint/aggs/UnsignedIntSumAggregatorFactory.java | 8 ++++++++ .../org/apache/druid/uint/UnsignedIntAggregationTest.java | 4 ++-- .../unsigned-int/src/test/resources/index_agg.json | 4 ++-- .../src/test/resources/queries/group_by_sum_query.json | 4 ++-- .../unsigned-int/src/test/resources/spec.json | 5 +++-- .../unsigned-int/src/test/resources/u_int_data.data | 4 ++-- 11 files changed, 42 insertions(+), 15 deletions(-) diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java index 13eb2766c027..007f455f9766 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java @@ -44,7 +44,8 @@ public Long fromByteBuffer(ByteBuffer buffer, int numBytes) buffer.get(cache); ByteBuffer result = ByteBuffer.allocate(8).put(new byte[]{0, 0, 0, 0}).put(cache); result.position(0); - return result.getLong(); + long toReturn = result.getLong(); + return toReturn; } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java index 753ebf8147b7..fa0567e3c5b9 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java @@ -43,19 +43,19 @@ public List getJacksonModules() .registerSubtypes( new NamedType( UnsignedIntSumAggregatorFactory.class, - UnsignedIntComplexSerde.TYPE + "unsigned_int_sum" ), new NamedType( UnsignedIntMaxAggregatorFactory.class, - UnsignedIntComplexSerde.TYPE + "unsigned_int_max" ), new NamedType( UnsignedIntMinAggregatorFactory.class, - UnsignedIntComplexSerde.TYPE + "unsigned_int_min" ), new NamedType( UnsignedIntAnyAggregatorFactory.class, - UnsignedIntComplexSerde.TYPE + "unsigned_int_any" ) ) ); diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java index e4c07993750c..88130c42dae2 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java @@ -45,4 +45,9 @@ public ColumnType getIntermediateType() return TYPE; } + @Override + public ColumnType getResultType() + { + return ColumnType.LONG; + } } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java index cabba5fae280..538e1f2f1937 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java @@ -51,4 +51,10 @@ public ColumnType getIntermediateType() return TYPE; } + @Override + public ColumnType getResultType() + { + return ColumnType.LONG; + } + } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java index 9a4b5bccc11b..0f92aac4a557 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java @@ -50,4 +50,10 @@ public ColumnType getIntermediateType() { return TYPE; } + + @Override + public ColumnType getResultType() + { + return ColumnType.LONG; + } } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java index 7ec1d0ada1bc..9543c7946638 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java @@ -23,8 +23,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.any.DoubleAnyAggregatorFactory; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.uint.UnsignedIntComplexSerde; import javax.annotation.Nullable; @@ -51,4 +54,9 @@ public ColumnType getIntermediateType() return TYPE; } + @Override + public ColumnType getResultType() + { + return TYPE; + } } diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java index ce8b90dd64e6..ebcf3b930d24 100644 --- a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java @@ -63,7 +63,7 @@ public void testSumAgg() throws Exception List expected = Lists.newArrayList( ResultRow.of("a", 199L), ResultRow.of("b", 98L), - ResultRow.of("c", (long) 2 * Integer.MAX_VALUE) // if only int agg, it would overflow. + ResultRow.of("c", 8589934590L) // if only int agg, it would overflow. ); assertEquals(3, actual.size()); assertArrayEquals(expected.toArray(), actual.toArray()); @@ -82,7 +82,7 @@ public void testMaxAgg() throws Exception List expected = Lists.newArrayList( ResultRow.of("a", 100L), ResultRow.of("b", 97L), - ResultRow.of("c", (long) Integer.MAX_VALUE) // if only int agg, it would overflow. + ResultRow.of("c", 0L) // if only int agg, it would overflow. ); assertEquals(3, actual.size()); assertArrayEquals(expected.toArray(), actual.toArray()); diff --git a/extensions-contrib/unsigned-int/src/test/resources/index_agg.json b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json index 339eba7e1988..e02f40fd0b0c 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/index_agg.json +++ b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json @@ -15,8 +15,8 @@ [ { - "name": "u_int", - "type": "unsigned_int", + "name": "foo", + "type": "unsigned_int_sum", "fieldName": "foo" } ] diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json index 891c83f6f0b7..c9241810a2c7 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json @@ -19,8 +19,8 @@ "granularity": "ALL", "aggregations": [ { - "type": "longSum", - "name": "out", + "type": "unsigned_int_sum", + "name": "foo", "fieldName": "foo" } ], diff --git a/extensions-contrib/unsigned-int/src/test/resources/spec.json b/extensions-contrib/unsigned-int/src/test/resources/spec.json index db62bb36e9c9..f3b71c4d43f0 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/spec.json +++ b/extensions-contrib/unsigned-int/src/test/resources/spec.json @@ -22,11 +22,12 @@ "format": "auto" }, "dimensionsSpec": { - "dimensions": [], + "dimensions": ["dim"], "dimensionExclusions": [ - "timestamp" + "timestamp", "foo" ], "spatialDimensions": [] } } + } diff --git a/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data index 6d26d1f981f5..63f1eb908784 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data +++ b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data @@ -2,5 +2,5 @@ {"timestamp": 101, "dim": "a", "foo": 99} {"timestamp": 102, "dim": "b", "foo": 97} {"timestamp": 104, "dim": "b", "foo": 1} -{"timestamp": 104, "dim": "c", "foo": 2147483647} -{"timestamp": 105, "dim": "c", "foo": 2147483647} \ No newline at end of file +{"timestamp": 104, "dim": "c", "foo": 4294967295} +{"timestamp": 105, "dim": "c", "foo": 4294967295} From 5170935eab2fa4dd54eee22b6475e922d841b321 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Fri, 18 Nov 2022 09:20:21 -0800 Subject: [PATCH 07/10] Fixing up tests and making things work --- .../apache/druid/uint/UnsignedIntDruidModule.java | 8 ++++---- .../aggs/UnsignedIntAnyAggregatorFactory.java | 5 ----- .../aggs/UnsignedIntMaxAggregatorFactory.java | 5 ----- .../aggs/UnsignedIntMinAggregatorFactory.java | 5 ----- .../aggs/UnsignedIntSumAggregatorFactory.java | 5 ----- .../druid/uint/UnsignedIntAggregationTest.java | 15 +++++++++------ .../src/test/resources/index_agg.json | 2 +- .../resources/queries/group_by_max_query.json | 2 +- .../resources/queries/group_by_min_query.json | 2 +- .../resources/queries/group_by_sum_query.json | 2 +- .../src/test/resources/u_int_data.data | 2 ++ 11 files changed, 19 insertions(+), 34 deletions(-) diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java index fa0567e3c5b9..f7cee4cc35d5 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java @@ -43,19 +43,19 @@ public List getJacksonModules() .registerSubtypes( new NamedType( UnsignedIntSumAggregatorFactory.class, - "unsigned_int_sum" + "unsignedIntSum" ), new NamedType( UnsignedIntMaxAggregatorFactory.class, - "unsigned_int_max" + "unsignedIntMax" ), new NamedType( UnsignedIntMinAggregatorFactory.class, - "unsigned_int_min" + "unsignedIntMin" ), new NamedType( UnsignedIntAnyAggregatorFactory.class, - "unsigned_int_any" + "unsignedIntAny" ) ) ); diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java index 88130c42dae2..e4c07993750c 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java @@ -45,9 +45,4 @@ public ColumnType getIntermediateType() return TYPE; } - @Override - public ColumnType getResultType() - { - return ColumnType.LONG; - } } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java index 538e1f2f1937..1ee7b936be1a 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java @@ -51,10 +51,5 @@ public ColumnType getIntermediateType() return TYPE; } - @Override - public ColumnType getResultType() - { - return ColumnType.LONG; - } } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java index 0f92aac4a557..03f3e9c27759 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java @@ -51,9 +51,4 @@ public ColumnType getIntermediateType() return TYPE; } - @Override - public ColumnType getResultType() - { - return ColumnType.LONG; - } } diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java index 9543c7946638..116b93628fe8 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java @@ -54,9 +54,4 @@ public ColumnType getIntermediateType() return TYPE; } - @Override - public ColumnType getResultType() - { - return TYPE; - } } diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java index ebcf3b930d24..0a855f956e5c 100644 --- a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java @@ -63,9 +63,10 @@ public void testSumAgg() throws Exception List expected = Lists.newArrayList( ResultRow.of("a", 199L), ResultRow.of("b", 98L), - ResultRow.of("c", 8589934590L) // if only int agg, it would overflow. + ResultRow.of("c", 8589934590L), // if only int agg, it would overflow. + ResultRow.of("d", 0L) // if only int agg, it would overflow. ); - assertEquals(3, actual.size()); + assertEquals(4, actual.size()); assertArrayEquals(expected.toArray(), actual.toArray()); } @@ -82,9 +83,10 @@ public void testMaxAgg() throws Exception List expected = Lists.newArrayList( ResultRow.of("a", 100L), ResultRow.of("b", 97L), - ResultRow.of("c", 0L) // if only int agg, it would overflow. + ResultRow.of("c", 4294967295L), // if only int agg, it would overflow. + ResultRow.of("d", 0L) // if only int agg, it would overflow. ); - assertEquals(3, actual.size()); + assertEquals(4, actual.size()); assertArrayEquals(expected.toArray(), actual.toArray()); } @@ -101,9 +103,10 @@ public void testMinAgg() throws Exception List expected = Lists.newArrayList( ResultRow.of("a", 99L), ResultRow.of("b", 1L), - ResultRow.of("c", (long) Integer.MAX_VALUE) // if only int agg, it would overflow. + ResultRow.of("c", 4294967295L), + ResultRow.of("d", 0L) // if only int agg, it would overflow. ); - assertEquals(3, actual.size()); + assertEquals(4, actual.size()); assertArrayEquals(expected.toArray(), actual.toArray()); } diff --git a/extensions-contrib/unsigned-int/src/test/resources/index_agg.json b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json index e02f40fd0b0c..4ab78db23eeb 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/index_agg.json +++ b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json @@ -16,7 +16,7 @@ [ { "name": "foo", - "type": "unsigned_int_sum", + "type": "unsignedIntSum", "fieldName": "foo" } ] diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json index 572edf63e1cb..ff9bc7c4c71c 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json @@ -19,7 +19,7 @@ "granularity": "ALL", "aggregations": [ { - "type": "longMax", + "type": "unsignedIntMax", "name": "out", "fieldName": "foo" } diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json index 499f46c5aba2..e06304fac53f 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json @@ -19,7 +19,7 @@ "granularity": "ALL", "aggregations": [ { - "type": "longMin", + "type": "unsignedIntMin", "name": "out", "fieldName": "foo" } diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json index c9241810a2c7..2590cbd8c5cb 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json @@ -19,7 +19,7 @@ "granularity": "ALL", "aggregations": [ { - "type": "unsigned_int_sum", + "type": "unsignedIntSum", "name": "foo", "fieldName": "foo" } diff --git a/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data index 63f1eb908784..7fbf6d0f7776 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data +++ b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data @@ -4,3 +4,5 @@ {"timestamp": 104, "dim": "b", "foo": 1} {"timestamp": 104, "dim": "c", "foo": 4294967295} {"timestamp": 105, "dim": "c", "foo": 4294967295} +{"timestamp": 106, "dim": "d", "foo": 4294967296} +{"timestamp": 107, "dim": "d", "foo": 4294967296} From 22661f41fe9b48e6289f33f99819f8f0233cf623 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Fri, 18 Nov 2022 10:05:47 -0800 Subject: [PATCH 08/10] Checkstyle issues --- .../druid/uint/aggs/UnsignedIntSumAggregatorFactory.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java index 116b93628fe8..7ec1d0ada1bc 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java @@ -23,11 +23,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.math.expr.ExprMacroTable; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.any.DoubleAnyAggregatorFactory; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.ValueType; import org.apache.druid.uint.UnsignedIntComplexSerde; import javax.annotation.Nullable; From d382f38525c433af005f86874378bbf5b64ba92b Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Tue, 29 Nov 2022 09:45:44 -0800 Subject: [PATCH 09/10] Using the updated APIs and added better tests --- .../java/org/apache/druid/uint/UnsignedIntComplexSerde.java | 1 + .../src/test/java/org/apache/druid/uint/DruidBaseTest.java | 2 +- extensions-contrib/unsigned-int/src/test/resources/spec.json | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java index 5493cfc3ed96..875096e7d633 100644 --- a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java @@ -74,4 +74,5 @@ public ObjectStrategy getObjectStrategy() { return strategy; } + } diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java index 0a51e920e695..be8052511d28 100644 --- a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java @@ -76,7 +76,7 @@ public static void createSegmentFromDataFile( segmentDir, 0, Granularities.NONE, - 2, + 1, false ); } diff --git a/extensions-contrib/unsigned-int/src/test/resources/spec.json b/extensions-contrib/unsigned-int/src/test/resources/spec.json index f3b71c4d43f0..dfc8a9285a75 100644 --- a/extensions-contrib/unsigned-int/src/test/resources/spec.json +++ b/extensions-contrib/unsigned-int/src/test/resources/spec.json @@ -22,7 +22,6 @@ "format": "auto" }, "dimensionsSpec": { - "dimensions": ["dim"], "dimensionExclusions": [ "timestamp", "foo" ], From fe74cc9fac8302b56ebdcb4c7da761050c8b9394 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Tue, 29 Nov 2022 09:55:53 -0800 Subject: [PATCH 10/10] Github actions fails to build, looks like its pulling from master, not the branch so had to rebase and force push, lets see if this works --- extensions-contrib/unsigned-int/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/unsigned-int/pom.xml b/extensions-contrib/unsigned-int/pom.xml index 9cc9e4d6493b..a9a80e03515a 100644 --- a/extensions-contrib/unsigned-int/pom.xml +++ b/extensions-contrib/unsigned-int/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 25.0.0-SNAPSHOT + 26.0.0-SNAPSHOT ../../pom.xml