diff --git a/docs/development/extensions-contrib/unsigned-int.md b/docs/development/extensions-contrib/unsigned-int.md new file mode 100644 index 000000000000..cbfc5fe03205 --- /dev/null +++ b/docs/development/extensions-contrib/unsigned-int.md @@ -0,0 +1,47 @@ +--- +id: unsigned-int +title: "Unsigned Integer Column Type" +--- + + + +Apache Druid Extension to utilize an unsigned int column. + +Consider this an [EXPERIMENTAL](../experimental.md) feature mostly because it has not been tested yet on a wide variety of long-running Druid clusters. + +## How it works + +Only store a 4 byte unsigned integer instead of a long column when you need your columns don't require you to store a long. Aggregations still happen as long values, only the data stored on disk is as an unsigned integer. + +## Configuration + +To use this extension please make sure to [include](../extensions.md#loading-extensions)`unsigned-int-extensions` in the extensions load list. + +Example Usage: + +``` +"metricsSpec": [ + { + "type": "unsigned_int", + "name": "value", + "fieldName": "value" + }, +``` + diff --git a/docs/development/extensions.md b/docs/development/extensions.md index 36d3549b195e..b165f88f3b75 100644 --- a/docs/development/extensions.md +++ b/docs/development/extensions.md @@ -98,6 +98,7 @@ All of these community extensions can be downloaded using [pull-deps](../operati |gce-extensions|GCE Extensions|[link](../development/extensions-contrib/gce-extensions.md)| |prometheus-emitter|Exposes [Druid metrics](../operations/metrics.md) for Prometheus server collection (https://prometheus.io/)|[link](./extensions-contrib/prometheus.md)| |kubernetes-overlord-extensions|Support for launching tasks in k8s without Middle Managers|[link](../development/extensions-contrib/k8s-jobs.md)| +|unsigned-int-extensions|Support for an unsigned integer column|[link](../development/extensions-contrib/unsigned-int.md)| ## Promoting community extensions to core extensions diff --git a/extensions-contrib/unsigned-int/pom.xml b/extensions-contrib/unsigned-int/pom.xml new file mode 100644 index 000000000000..a9a80e03515a --- /dev/null +++ b/extensions-contrib/unsigned-int/pom.xml @@ -0,0 +1,98 @@ + + + + + 4.0.0 + + org.apache.druid.extensions.contrib + unsigned-int-extensions + unsigned-int-extensions + unsigned-int-extensions + + + org.apache.druid + druid + 26.0.0-SNAPSHOT + ../../pom.xml + + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + org.apache.druid + druid-processing + ${project.parent.version} + provided + + + com.google.code.findbugs + jsr305 + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.google.guava + guava + provided + + + com.google.inject + guice + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + org.apache.druid + druid-processing + ${project.parent.version} + test-jar + test + + + org.apache.druid + druid-core + ${project.parent.version} + test-jar + test + + + junit + junit + test + + + + + diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java new file mode 100644 index 000000000000..007f455f9766 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnSignedIntObjectStrategy.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import org.apache.druid.segment.data.ObjectStrategy; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class UnSignedIntObjectStrategy implements ObjectStrategy +{ + @Override + public Class getClazz() + { + return Long.class; + } + + @Nullable + @Override + public Long fromByteBuffer(ByteBuffer buffer, int numBytes) + { + if (numBytes <= 0) { + return null; + } + byte[] cache = new byte[numBytes]; + buffer.get(cache); + ByteBuffer result = ByteBuffer.allocate(8).put(new byte[]{0, 0, 0, 0}).put(cache); + result.position(0); + long toReturn = result.getLong(); + return toReturn; + + } + + @Nullable + @Override + public byte[] toBytes(@Nullable Long val) + { + if (val == null) { + return new byte[0]; + } + byte[] bytes = new byte[8]; + ByteBuffer.wrap(bytes).putLong(val); + return Arrays.copyOfRange(bytes, 4, 8); + } + + @Override + public int compare(Long left, Long right) + { + return left.compareTo(right); + } +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java new file mode 100644 index 000000000000..875096e7d633 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntComplexSerde.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.ComplexColumnPartSupplier; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.apache.druid.segment.serde.ComplexMetricSerde; + +import java.nio.ByteBuffer; + +public class UnsignedIntComplexSerde extends ComplexMetricSerde +{ + + public static final String TYPE = "unsigned_int"; + private final UnSignedIntObjectStrategy strategy = new UnSignedIntObjectStrategy(); + + + @Override + public String getTypeName() + { + return TYPE; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + return new ComplexMetricExtractor() + { + @Override + public Class extractedClass() + { + return Long.class; + } + + @Override + public Long extractValue(InputRow inputRow, String metricName) + { + Object obj = inputRow.getRaw(metricName); + return (Long) obj; + } + }; + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder) + { + GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), builder.getFileMapper()); + builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return strategy; + } + +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java new file mode 100644 index 000000000000..f7cee4cc35d5 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/UnsignedIntDruidModule.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.segment.serde.ComplexMetrics; +import org.apache.druid.uint.aggs.UnsignedIntAnyAggregatorFactory; +import org.apache.druid.uint.aggs.UnsignedIntMaxAggregatorFactory; +import org.apache.druid.uint.aggs.UnsignedIntMinAggregatorFactory; +import org.apache.druid.uint.aggs.UnsignedIntSumAggregatorFactory; + +import java.util.Collections; +import java.util.List; + +public class UnsignedIntDruidModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return Collections.singletonList( + new SimpleModule("UnsignedIntDruidModule") + .registerSubtypes( + new NamedType( + UnsignedIntSumAggregatorFactory.class, + "unsignedIntSum" + ), + new NamedType( + UnsignedIntMaxAggregatorFactory.class, + "unsignedIntMax" + ), + new NamedType( + UnsignedIntMinAggregatorFactory.class, + "unsignedIntMin" + ), + new NamedType( + UnsignedIntAnyAggregatorFactory.class, + "unsignedIntAny" + ) + ) + ); + } + + @Override + public void configure(Binder binder) + { + if (ComplexMetrics.getSerdeForType(UnsignedIntComplexSerde.TYPE) == null) { + ComplexMetrics.registerSerde(UnsignedIntComplexSerde.TYPE, new UnsignedIntComplexSerde()); + } + } +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java new file mode 100644 index 000000000000..e4c07993750c --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntAnyAggregatorFactory.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint.aggs; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.uint.UnsignedIntComplexSerde; + +public class UnsignedIntAnyAggregatorFactory extends LongAnyAggregatorFactory +{ + + public static final ColumnType TYPE = ColumnType.ofComplex(UnsignedIntComplexSerde.TYPE); + + @JsonCreator + public UnsignedIntAnyAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") final String fieldName + ) + { + super(name, fieldName); + } + + @Override + public ColumnType getIntermediateType() + { + return TYPE; + } + +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java new file mode 100644 index 000000000000..1ee7b936be1a --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMaxAggregatorFactory.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint.aggs; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.uint.UnsignedIntComplexSerde; + +import javax.annotation.Nullable; + +public class UnsignedIntMaxAggregatorFactory extends LongMaxAggregatorFactory +{ + + public static final ColumnType TYPE = ColumnType.ofComplex(UnsignedIntComplexSerde.TYPE); + + @JsonCreator + public UnsignedIntMaxAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("expression") @Nullable String expression, + @JacksonInject ExprMacroTable macroTable + ) + { + super(name, fieldName, expression, macroTable); + } + + @Override + public ColumnType getIntermediateType() + { + return TYPE; + } + + +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java new file mode 100644 index 000000000000..03f3e9c27759 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntMinAggregatorFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint.aggs; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.LongMinAggregatorFactory; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.uint.UnsignedIntComplexSerde; + +import javax.annotation.Nullable; + +public class UnsignedIntMinAggregatorFactory extends LongMinAggregatorFactory +{ + + public static final ColumnType TYPE = ColumnType.ofComplex(UnsignedIntComplexSerde.TYPE); + + @JsonCreator + public UnsignedIntMinAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("expression") @Nullable String expression, + @JacksonInject ExprMacroTable macroTable + ) + { + super(name, fieldName, expression, macroTable); + } + + @Override + public ColumnType getIntermediateType() + { + return TYPE; + } + +} diff --git a/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java new file mode 100644 index 000000000000..7ec1d0ada1bc --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/java/org/apache/druid/uint/aggs/UnsignedIntSumAggregatorFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint.aggs; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.uint.UnsignedIntComplexSerde; + +import javax.annotation.Nullable; + +public class UnsignedIntSumAggregatorFactory extends LongSumAggregatorFactory +{ + + public static final ColumnType TYPE = ColumnType.ofComplex(UnsignedIntComplexSerde.TYPE); + + @JsonCreator + public UnsignedIntSumAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("expression") @Nullable String expression, + @JacksonInject ExprMacroTable macroTable + ) + { + super(name, fieldName, expression, macroTable); + } + + @Override + public ColumnType getIntermediateType() + { + return TYPE; + } + +} diff --git a/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..197142de0469 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.uint.UnsignedIntDruidModule \ No newline at end of file diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java new file mode 100644 index 000000000000..be8052511d28 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/DruidBaseTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.aggregation.AggregationTestHelper; +import org.apache.druid.query.groupby.GroupByQueryConfig; +import org.apache.druid.query.groupby.strategy.GroupByStrategySelector; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Objects; + +public abstract class DruidBaseTest +{ + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + protected final AggregationTestHelper groupByTestHelper; + protected final ObjectMapper jsonMapper; + + public DruidBaseTest() + { + UnsignedIntDruidModule module = new UnsignedIntDruidModule(); + module.configure(null); + GroupByQueryConfig config = new GroupByQueryConfig() + { + @Override + public String getDefaultStrategy() + { + return GroupByStrategySelector.STRATEGY_V2; + } + }; + + groupByTestHelper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( + module.getJacksonModules(), config, tempFolder + ); + + jsonMapper = groupByTestHelper.getObjectMapper(); + } + + public static void createSegmentFromDataFile( + AggregationTestHelper aggregationTestHelper, + String dataFile, File segmentDir + ) throws Exception + { + aggregationTestHelper.createIndex( + Paths.get(Objects.requireNonNull(DruidBaseTest.class.getClassLoader().getResource(dataFile)) + .toURI()).toFile(), + readJsonAsString("spec.json"), + readJsonAsString("index_agg.json"), + segmentDir, + 0, + Granularities.NONE, + 1, + false + ); + } + + public static String readJsonAsString(String s) throws IOException, URISyntaxException + { + List lines = Files.readAllLines( + Paths + .get(Objects.requireNonNull(DruidBaseTest.class.getClassLoader() + .getResource( + s)) + .toURI())); + StringBuilder builder = new StringBuilder(); + for (String line : lines) { + if (!line.trim().startsWith("//")) { + builder.append(line).append(" "); + } + } + return builder.toString(); + } + +} diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnSignedIntObjectStrategyTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnSignedIntObjectStrategyTest.java new file mode 100644 index 000000000000..d72fe4cfc53b --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnSignedIntObjectStrategyTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class UnSignedIntObjectStrategyTest +{ + + @Test + public void testSimpleCase() + { + UnSignedIntObjectStrategy strategy = new UnSignedIntObjectStrategy(); + byte[] bytes = strategy.toBytes(2L); + assertEquals(4, bytes.length); + Long result = strategy.fromByteBuffer(ByteBuffer.wrap(bytes), bytes.length); + assertEquals(Long.valueOf(2), result); + } + + @Test + public void testNullObject() + { + UnSignedIntObjectStrategy strategy = new UnSignedIntObjectStrategy(); + byte[] bytes = strategy.toBytes(null); + Long result = strategy.fromByteBuffer(ByteBuffer.wrap(bytes), bytes.length); + assertNull(result); + } +} diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java new file mode 100644 index 000000000000..0a855f956e5c --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntAggregationTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.query.groupby.ResultRow; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class UnsignedIntAggregationTest extends DruidBaseTest +{ + + private File segmentDir; + + @Before + public void setUp() throws Exception + { + segmentDir = tempFolder.newFolder(); + } + + @After + public void tearDown() + { + segmentDir.delete(); + } + + @Test + public void testSumAgg() throws Exception + { + createSegmentFromDataFile(groupByTestHelper, "u_int_data.data", segmentDir); + Sequence sequence = groupByTestHelper.runQueryOnSegments( + ImmutableList.of(segmentDir), + readJsonAsString("queries/group_by_sum_query.json") + ); + + List actual = sequence.toList(); + List expected = Lists.newArrayList( + ResultRow.of("a", 199L), + ResultRow.of("b", 98L), + ResultRow.of("c", 8589934590L), // if only int agg, it would overflow. + ResultRow.of("d", 0L) // if only int agg, it would overflow. + ); + assertEquals(4, actual.size()); + assertArrayEquals(expected.toArray(), actual.toArray()); + } + + @Test + public void testMaxAgg() throws Exception + { + createSegmentFromDataFile(groupByTestHelper, "u_int_data.data", segmentDir); + Sequence sequence = groupByTestHelper.runQueryOnSegments( + ImmutableList.of(segmentDir), + readJsonAsString("queries/group_by_max_query.json") + ); + + List actual = sequence.toList(); + List expected = Lists.newArrayList( + ResultRow.of("a", 100L), + ResultRow.of("b", 97L), + ResultRow.of("c", 4294967295L), // if only int agg, it would overflow. + ResultRow.of("d", 0L) // if only int agg, it would overflow. + ); + assertEquals(4, actual.size()); + assertArrayEquals(expected.toArray(), actual.toArray()); + } + + @Test + public void testMinAgg() throws Exception + { + createSegmentFromDataFile(groupByTestHelper, "u_int_data.data", segmentDir); + Sequence sequence = groupByTestHelper.runQueryOnSegments( + ImmutableList.of(segmentDir), + readJsonAsString("queries/group_by_min_query.json") + ); + + List actual = sequence.toList(); + List expected = Lists.newArrayList( + ResultRow.of("a", 99L), + ResultRow.of("b", 1L), + ResultRow.of("c", 4294967295L), + ResultRow.of("d", 0L) // if only int agg, it would overflow. + ); + assertEquals(4, actual.size()); + assertArrayEquals(expected.toArray(), actual.toArray()); + } + +} diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntComplexSerdeTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntComplexSerdeTest.java new file mode 100644 index 000000000000..babad205e919 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntComplexSerdeTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.junit.Assert; +import org.junit.Test; + +public class UnsignedIntComplexSerdeTest +{ + @Test + public void testSerde() + { + final UnsignedIntComplexSerde serde = new UnsignedIntComplexSerde(); + final ComplexMetricExtractor extractor = serde.getExtractor(); + final Long value = (Long) extractor.extractValue( + new MapBasedInputRow(0L, ImmutableList.of(), ImmutableMap.of("foo", 3L)), + "foo" + ); + Assert.assertEquals(Long.valueOf(3), value); + } +} diff --git a/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntDruidModuleTest.java b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntDruidModuleTest.java new file mode 100644 index 000000000000..bdd1aaa9291f --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/java/org/apache/druid/uint/UnsignedIntDruidModuleTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.uint; + +import com.fasterxml.jackson.databind.Module; +import com.google.common.collect.Iterables; +import org.junit.Test; + +import java.util.List; + +import static junit.framework.Assert.assertEquals; + +public class UnsignedIntDruidModuleTest +{ + + @Test + public void testSomethingSimple() + { + List result = new UnsignedIntDruidModule().getJacksonModules(); + assertEquals(1, result.size()); + assertEquals("UnsignedIntDruidModule", Iterables.getOnlyElement(result).getModuleName()); + } + +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/index_agg.json b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json new file mode 100644 index 000000000000..4ab78db23eeb --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/index_agg.json @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +[ + { + "name": "foo", + "type": "unsignedIntSum", + "fieldName": "foo" + } +] diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json new file mode 100644 index 000000000000..ff9bc7c4c71c --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_max_query.json @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "queryType": "groupBy", + "dataSource": "test_datasource", + "granularity": "ALL", + "aggregations": [ + { + "type": "unsignedIntMax", + "name": "out", + "fieldName": "foo" + } + ], + "dimensions": [ + "dim" + ], + "intervals": [ + "1970-01-01T00:00:00.000Z/2020-10-22T00:00:00.000Z" + ] +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json new file mode 100644 index 000000000000..e06304fac53f --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_min_query.json @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "queryType": "groupBy", + "dataSource": "test_datasource", + "granularity": "ALL", + "aggregations": [ + { + "type": "unsignedIntMin", + "name": "out", + "fieldName": "foo" + } + ], + "dimensions": [ + "dim" + ], + "intervals": [ + "1970-01-01T00:00:00.000Z/2020-10-22T00:00:00.000Z" + ] +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json new file mode 100644 index 000000000000..2590cbd8c5cb --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/queries/group_by_sum_query.json @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "queryType": "groupBy", + "dataSource": "test_datasource", + "granularity": "ALL", + "aggregations": [ + { + "type": "unsignedIntSum", + "name": "foo", + "fieldName": "foo" + } + ], + "dimensions": [ + "dim" + ], + "intervals": [ + "1970-01-01T00:00:00.000Z/2020-10-22T00:00:00.000Z" + ] +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/spec.json b/extensions-contrib/unsigned-int/src/test/resources/spec.json new file mode 100644 index 000000000000..dfc8a9285a75 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/spec.json @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp", + "format": "auto" + }, + "dimensionsSpec": { + "dimensionExclusions": [ + "timestamp", "foo" + ], + "spatialDimensions": [] + } + } + +} diff --git a/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data new file mode 100644 index 000000000000..7fbf6d0f7776 --- /dev/null +++ b/extensions-contrib/unsigned-int/src/test/resources/u_int_data.data @@ -0,0 +1,8 @@ +{"timestamp": 100, "dim": "a", "foo": 100} +{"timestamp": 101, "dim": "a", "foo": 99} +{"timestamp": 102, "dim": "b", "foo": 97} +{"timestamp": 104, "dim": "b", "foo": 1} +{"timestamp": 104, "dim": "c", "foo": 4294967295} +{"timestamp": 105, "dim": "c", "foo": 4294967295} +{"timestamp": 106, "dim": "d", "foo": 4294967296} +{"timestamp": 107, "dim": "d", "foo": 4294967296} diff --git a/pom.xml b/pom.xml index 5e44487413d2..da3fe1b7e5ed 100644 --- a/pom.xml +++ b/pom.xml @@ -215,6 +215,7 @@ extensions-contrib/prometheus-emitter extensions-contrib/opentelemetry-emitter extensions-contrib/kubernetes-overlord-extensions + extensions-contrib/unsigned-int distribution diff --git a/website/.spelling b/website/.spelling index d02f4f846799..436468e5c65c 100644 --- a/website/.spelling +++ b/website/.spelling @@ -506,6 +506,7 @@ unnests unparseable unparsed unsetting +unsigned untrusted useFilterCNF useJqSyntax