From bfe595c4f7c732b056b968823cafc111e7cebdef Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 15 Nov 2015 00:33:36 -0600 Subject: [PATCH 1/4] removing unused antlr grammer etc --- server/pom.xml | 15 - .../antlr4/io/druid/sql/antlr4/DruidSQL.g4 | 343 ------------------ .../java/io/druid/server/sql/SQLRunner.java | 227 ------------ 3 files changed, 585 deletions(-) delete mode 100644 server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 delete mode 100644 server/src/main/java/io/druid/server/sql/SQLRunner.java diff --git a/server/pom.xml b/server/pom.xml index cf27dbd902f2..4e0970c3e11e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -132,10 +132,6 @@ org.eclipse.aether aether-api - - org.antlr - antlr4-runtime - net.spy spymemcached @@ -222,17 +218,6 @@ - - org.antlr - antlr4-maven-plugin - - - - antlr4 - - - - diff --git a/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 b/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 deleted file mode 100644 index f86a252fdb62..000000000000 --- a/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 +++ /dev/null @@ -1,343 +0,0 @@ -grammar DruidSQL; - -@header { -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import io.druid.granularity.PeriodGranularity; -import io.druid.granularity.QueryGranularity; -import io.druid.query.aggregation.AggregatorFactory; -import io.druid.query.aggregation.CountAggregatorFactory; -import io.druid.query.aggregation.DoubleSumAggregatorFactory; -import io.druid.query.aggregation.DoubleMaxAggregatorFactory; -import io.druid.query.aggregation.DoubleMinAggregatorFactory; -import io.druid.query.aggregation.PostAggregator; -import io.druid.query.aggregation.post.ArithmeticPostAggregator; -import io.druid.query.aggregation.post.ConstantPostAggregator; -import io.druid.query.aggregation.post.FieldAccessPostAggregator; -import io.druid.query.dimension.DefaultDimensionSpec; -import io.druid.query.dimension.DimensionSpec; -import io.druid.query.filter.AndDimFilter; -import io.druid.query.filter.DimFilter; -import io.druid.query.filter.NotDimFilter; -import io.druid.query.filter.OrDimFilter; -import io.druid.query.filter.RegexDimFilter; -import io.druid.query.filter.SelectorDimFilter; -import org.antlr.v4.runtime.NoViableAltException; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNSimulator; -import org.antlr.v4.runtime.atn.ParserATNSimulator; -import org.antlr.v4.runtime.atn.PredictionContextCache; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.tree.ParseTreeListener; -import org.antlr.v4.runtime.tree.TerminalNode; -import org.joda.time.DateTime; -import org.joda.time.Period; - -import java.text.NumberFormat; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -} - -@parser::members { - public Map aggregators = new LinkedHashMap(); - public List postAggregators = new LinkedList(); - public DimFilter filter; - public List intervals; - public List fields = new LinkedList(); - public QueryGranularity granularity = QueryGranularity.ALL; - public Map groupByDimensions = new LinkedHashMap(); - - String dataSourceName = null; - - public String getDataSource() { - return dataSourceName; - } - - public String unescape(String quoted) { - String unquote = quoted.trim().replaceFirst("^'(.*)'\$", "\$1"); - return unquote.replace("''", "'"); - } - - AggregatorFactory evalAgg(String name, int fn) { - switch (fn) { - case SUM: return new DoubleSumAggregatorFactory("sum("+name+")", name); - case MIN: return new DoubleMinAggregatorFactory("min("+name+")", name); - case MAX: return new DoubleMaxAggregatorFactory("max("+name+")", name); - case COUNT: return new CountAggregatorFactory(name); - } - throw new IllegalArgumentException("Unknown function [" + fn + "]"); - } - - PostAggregator evalArithmeticPostAggregator(PostAggregator a, List ops, List b) { - if(b.isEmpty()) return a; - else { - int i = 0; - - PostAggregator root = a; - while(i < ops.size()) { - List list = new LinkedList(); - List names = new LinkedList(); - - names.add(root.getName()); - list.add(root); - - Token op = ops.get(i); - - while(i < ops.size() && ops.get(i).getType() == op.getType()) { - PostAggregator e = b.get(i); - list.add(e); - names.add(e.getName()); - i++; - } - - root = new ArithmeticPostAggregator("("+Joiner.on(op.getText()).join(names)+")", op.getText(), list); - } - - return root; - } - } -} - - -AND: 'and'; -OR: 'or'; -SUM: 'sum'; -MIN: 'min'; -MAX: 'max'; -COUNT: 'count'; -AS: 'as'; -OPEN: '('; -CLOSE: ')'; -STAR: '*'; -NOT: '!' ; -PLUS: '+'; -MINUS: '-'; -DIV: '/'; -COMMA: ','; -EQ: '='; -NEQ: '!='; -MATCH: '~'; -GROUP: 'group'; - -IDENT : (LETTER)(LETTER | DIGIT | '_')* ; -QUOTED_STRING : '\'' ( ESC | ~'\'' )*? '\'' ; -ESC : '\'' '\''; - -NUMBER: DIGIT*'.'?DIGIT+(EXPONENT)?; -EXPONENT: ('e') ('+'|'-')? ('0'..'9')+; -fragment DIGIT : '0'..'9'; -fragment LETTER : 'a'..'z' | 'A'..'Z'; - -LINE_COMMENT : '--' .*? '\r'? '\n' -> skip ; -COMMENT : '/*' .*? '*/' -> skip ; -WS : (' '| '\t' | '\r' '\n' | '\n' | '\r')+ -> skip; - - - -query - : select_stmt where_stmt (groupby_stmt)? - ; - -select_stmt - : 'select' e+=aliasedExpression (',' e+=aliasedExpression)* 'from' datasource { - for(AliasedExpressionContext a : $e) { - postAggregators.add(a.p); - fields.add(a.p.getName()); - } - this.dataSourceName = $datasource.text; - } - ; - -where_stmt - : 'where' f=timeAndDimFilter { - if($f.filter != null) this.filter = $f.filter; - this.intervals = Lists.newArrayList($f.interval); - } - ; - -groupby_stmt - : GROUP 'by' groupByExpression ( COMMA! groupByExpression )* - ; - -groupByExpression - : gran=granularityFn {this.granularity = $gran.granularity;} - | dim=IDENT { this.groupByDimensions.put($dim.text, new DefaultDimensionSpec($dim.text, $dim.text)); } - ; - -datasource - : IDENT - ; - -aliasedExpression returns [PostAggregator p] - : expression ( AS^ name=IDENT )? { - if($name != null) { - postAggregators.add($expression.p); - $p = new FieldAccessPostAggregator($name.text, $expression.p.getName()); - } - else $p = $expression.p; - } - ; - -expression returns [PostAggregator p] - : additiveExpression { $p = $additiveExpression.p; } - ; - -additiveExpression returns [PostAggregator p] - : a=multiplyExpression (( ops+=PLUS^ | ops+=MINUS^ ) b+=multiplyExpression)* { - List rhs = new LinkedList(); - for(MultiplyExpressionContext e : $b) rhs.add(e.p); - $p = evalArithmeticPostAggregator($a.p, $ops, rhs); - } - ; - -multiplyExpression returns [PostAggregator p] - : a=unaryExpression ((ops+= STAR | ops+=DIV ) b+=unaryExpression)* { - List rhs = new LinkedList(); - for(UnaryExpressionContext e : $b) rhs.add(e.p); - $p = evalArithmeticPostAggregator($a.p, $ops, rhs); - } - ; - -unaryExpression returns [PostAggregator p] - : MINUS e=unaryExpression { - if($e.p instanceof ConstantPostAggregator) { - ConstantPostAggregator c = (ConstantPostAggregator)$e.p; - double v = c.getConstantValue().doubleValue() * -1; - $p = new ConstantPostAggregator(Double.toString(v), v); - } else { - $p = new ArithmeticPostAggregator( - "-"+$e.p.getName(), - "*", - Lists.newArrayList($e.p, new ConstantPostAggregator("-1", -1.0)) - ); - } - } - | PLUS e=unaryExpression { $p = $e.p; } - | primaryExpression { $p = $primaryExpression.p; } - ; - -primaryExpression returns [PostAggregator p] - : constant { $p = $constant.c; } - | aggregate { - aggregators.put($aggregate.agg.getName(), $aggregate.agg); - $p = new FieldAccessPostAggregator($aggregate.agg.getName(), $aggregate.agg.getName()); - } - | OPEN! e=expression CLOSE! { $p = $e.p; } - ; - -aggregate returns [AggregatorFactory agg] - : fn=( SUM^ | MIN^ | MAX^ ) OPEN! name=(IDENT|COUNT) CLOSE! { $agg = evalAgg($name.text, $fn.type); } - | fn=COUNT OPEN! STAR CLOSE! { $agg = evalAgg("count(*)", $fn.type); } - ; - -constant returns [ConstantPostAggregator c] - : value=NUMBER { double v = Double.parseDouble($value.text); $c = new ConstantPostAggregator(Double.toString(v), v); } - ; - -/* time filters must be top level filters */ -timeAndDimFilter returns [DimFilter filter, org.joda.time.Interval interval] - : (f1=dimFilter AND)? t=timeFilter (AND f2=dimFilter)? { - if($f1.ctx != null || $f2.ctx != null) { - if($f1.ctx != null && $f2.ctx != null) { - $filter = new AndDimFilter(Lists.newArrayList($f1.filter, $f2.filter)); - } else if($f1.ctx != null) { - $filter = $f1.filter; - } else { - $filter = $f2.filter; - } - } - $interval = $t.interval; - } - ; - -dimFilter returns [DimFilter filter] - : e=orDimFilter { $filter = $e.filter; } - ; - -orDimFilter returns [DimFilter filter] - : a=andDimFilter (OR^ b+=andDimFilter)* { - if($b.isEmpty()) $filter = $a.filter; - else { - List rest = new ArrayList(); - for(AndDimFilterContext e : $b) rest.add(e.filter); - $filter = new OrDimFilter(Lists.asList($a.filter, rest.toArray(new DimFilter[]{}))); - } - } - ; - -andDimFilter returns [DimFilter filter] - : a=primaryDimFilter (AND^ b+=primaryDimFilter)* { - if($b.isEmpty()) $filter = $a.filter; - else { - List rest = new ArrayList(); - for(PrimaryDimFilterContext e : $b) rest.add(e.filter); - $filter = new AndDimFilter(Lists.asList($a.filter, rest.toArray(new DimFilter[]{}))); - } - } - ; - -primaryDimFilter returns [DimFilter filter] - : e=selectorDimFilter { $filter = $e.filter; } - | l=inListDimFilter { $filter = $l.filter; } - | NOT f=dimFilter { $filter = new NotDimFilter($f.filter); } - | OPEN! f=dimFilter CLOSE! { $filter = $f.filter; } - ; - -selectorDimFilter returns [DimFilter filter] - : dimension=IDENT op=(EQ|NEQ|MATCH) value=QUOTED_STRING { - String dim = $dimension.text; - String val = unescape($value.text); - switch($op.type) { - case(EQ): $filter = new SelectorDimFilter(dim, val); break; - case(NEQ): $filter = new NotDimFilter(new SelectorDimFilter(dim, val)); break; - case(MATCH): $filter = new RegexDimFilter(dim, val); break; - } - } - ; - -inListDimFilter returns [DimFilter filter] - : dimension=IDENT 'in' (OPEN! ( (list+=QUOTED_STRING (COMMA! list+=QUOTED_STRING)*) ) CLOSE!) { - List filterList = new LinkedList(); - for(Token e : $list) filterList.add(new SelectorDimFilter($dimension.text, unescape(e.getText()))); - $filter = new OrDimFilter(filterList); - } - ; - -timeFilter returns [org.joda.time.Interval interval, QueryGranularity granularity] - : 'timestamp' 'between' s=timestamp AND e=timestamp { - $interval = new org.joda.time.Interval($s.t, $e.t); - } - ; - -granularityFn returns [QueryGranularity granularity] - : 'granularity' OPEN! 'timestamp' ',' str=QUOTED_STRING CLOSE! { - String granStr = unescape($str.text); - try { - $granularity = QueryGranularity.fromString(granStr); - } catch(IllegalArgumentException e) { - $granularity = new PeriodGranularity(new Period(granStr), null, null); - } - } - ; - -timestamp returns [DateTime t] - : NUMBER { - String str = $NUMBER.text.trim(); - try { - $t = new DateTime(NumberFormat.getInstance().parse(str)); - } - catch(ParseException e) { - throw new IllegalArgumentException("Unable to parse number [" + str + "]"); - } - } - | QUOTED_STRING { $t = new DateTime(unescape($QUOTED_STRING.text)); } - ; diff --git a/server/src/main/java/io/druid/server/sql/SQLRunner.java b/server/src/main/java/io/druid/server/sql/SQLRunner.java deleted file mode 100644 index b578636c32bd..000000000000 --- a/server/src/main/java/io/druid/server/sql/SQLRunner.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright 2012 - 2015 Metamarkets Group Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.druid.server.sql; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.base.Charsets; -import com.google.common.base.Function; -import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.metamx.common.StringUtils; -import com.metamx.common.guava.CloseQuietly; -import io.druid.data.input.Row; -import io.druid.jackson.DefaultObjectMapper; -import io.druid.query.Druids; -import io.druid.query.Query; -import io.druid.query.Result; -import io.druid.query.aggregation.AggregatorFactory; -import io.druid.query.dimension.DimensionSpec; -import io.druid.query.groupby.GroupByQuery; -import io.druid.query.timeseries.TimeseriesResultValue; -import io.druid.sql.antlr4.DruidSQLLexer; -import io.druid.sql.antlr4.DruidSQLParser; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ConsoleErrorListener; -import org.antlr.v4.runtime.TokenStream; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; - -import javax.annotation.Nullable; -import javax.ws.rs.core.MediaType; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.List; - -public class SQLRunner -{ - private static final String STATEMENT = "select count(*), (1 - count(*) / sum(count)) * 100 as ratio from wikipedia where" - + " timestamp between '2013-02-01' and '2013-02-14'" - + " and (namespace = 'article' or page ~ 'Talk:.*')" - + " and language in ( 'en', 'fr' ) " - + " and user ~ '(?i)^david.*'" - + " group by granularity(timestamp, 'day'), language"; - - public static void main(String[] args) throws Exception - { - - Options options = new Options(); - options.addOption("h", "help", false, "help"); - options.addOption("v", false, "verbose"); - options.addOption("e", "host", true, "endpoint [hostname:port]"); - - CommandLine cmd = new GnuParser().parse(options, args); - - if(cmd.hasOption("h")) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("SQLRunner", options); - System.exit(2); - } - - String hostname = cmd.getOptionValue("e", "localhost:8080"); - String sql = cmd.getArgs().length > 0 ? cmd.getArgs()[0] : STATEMENT; - - ObjectMapper objectMapper = new DefaultObjectMapper(); - ObjectWriter jsonWriter = objectMapper.writerWithDefaultPrettyPrinter(); - - CharStream stream = new ANTLRInputStream(sql); - DruidSQLLexer lexer = new DruidSQLLexer(stream); - TokenStream tokenStream = new CommonTokenStream(lexer); - DruidSQLParser parser = new DruidSQLParser(tokenStream); - lexer.removeErrorListeners(); - parser.removeErrorListeners(); - - lexer.addErrorListener(ConsoleErrorListener.INSTANCE); - parser.addErrorListener(ConsoleErrorListener.INSTANCE); - - try { - DruidSQLParser.QueryContext queryContext = parser.query(); - if(parser.getNumberOfSyntaxErrors() > 0) throw new IllegalStateException(); -// parser.setBuildParseTree(true); -// System.err.println(q.toStringTree(parser)); - } catch(Exception e) { - String msg = e.getMessage(); - if(msg != null) System.err.println(e); - System.exit(1); - } - - final Query query; - final TypeReference typeRef; - boolean groupBy = false; - if(parser.groupByDimensions.isEmpty()) { - query = Druids.newTimeseriesQueryBuilder() - .dataSource(parser.getDataSource()) - .aggregators(new ArrayList(parser.aggregators.values())) - .postAggregators(parser.postAggregators) - .intervals(parser.intervals) - .granularity(parser.granularity) - .filters(parser.filter) - .build(); - - typeRef = new TypeReference>>(){}; - } else { - query = GroupByQuery.builder() - .setDataSource(parser.getDataSource()) - .setAggregatorSpecs(new ArrayList(parser.aggregators.values())) - .setPostAggregatorSpecs(parser.postAggregators) - .setInterval(parser.intervals) - .setGranularity(parser.granularity) - .setDimFilter(parser.filter) - .setDimensions(new ArrayList(parser.groupByDimensions.values())) - .build(); - - typeRef = new TypeReference>(){}; - groupBy = true; - } - - String queryStr = jsonWriter.writeValueAsString(query); - if(cmd.hasOption("v")) System.err.println(queryStr); - - URL url = new URL(String.format("http://%s/druid/v2/?pretty", hostname)); - final URLConnection urlConnection = url.openConnection(); - urlConnection.addRequestProperty("content-type", MediaType.APPLICATION_JSON); - urlConnection.getOutputStream().write(StringUtils.toUtf8(queryStr)); - BufferedReader stdInput = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), Charsets.UTF_8)); - - Object res = objectMapper.readValue(stdInput, typeRef); - - Joiner tabJoiner = Joiner.on("\t"); - - if(groupBy) { - List rows = (List)res; - Iterable dimensions = Iterables.transform(parser.groupByDimensions.values(), new Function() - { - @Override - public String apply(@Nullable DimensionSpec input) - { - return input.getOutputName(); - } - }); - - System.out.println(tabJoiner.join(Iterables.concat( - Lists.newArrayList("timestamp"), - dimensions, - parser.fields - ))); - for(final Row r : rows) { - System.out.println( - tabJoiner.join( - Iterables.concat( - Lists.newArrayList(parser.granularity.toDateTime(r.getTimestampFromEpoch())), - Iterables.transform( - parser.groupByDimensions.values(), new Function() - { - @Override - public String apply(@Nullable DimensionSpec input) - { - return Joiner.on(",").join(r.getDimension(input.getOutputName())); - } - }), - Iterables.transform(parser.fields, new Function() - { - @Override - public Object apply(@Nullable String input) - { - return r.getFloatMetric(input); - } - }) - ) - ) - ); - } - } - else { - List> rows = (List>)res; - System.out.println(tabJoiner.join(Iterables.concat( - Lists.newArrayList("timestamp"), - parser.fields - ))); - for(final Result r : rows) { - System.out.println( - tabJoiner.join( - Iterables.concat( - Lists.newArrayList(r.getTimestamp()), - Lists.transform( - parser.fields, - new Function() - { - @Override - public Object apply(@Nullable String input) - { - return r.getValue().getMetric(input); - } - } - ) - ) - ) - ); - } - } - - CloseQuietly.close(stdInput); - } -} From ee326a8cf49d340f6b487837c342a49ce0d1d9d4 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 15 Nov 2015 00:26:20 -0600 Subject: [PATCH 2/4] adding attribute "exponent" to [long/double]Sum aggregators --- .../indexer/BatchDeltaIngestionTest.java | 2 +- .../DetermineHashedPartitionsJobTest.java | 2 +- .../indexer/DeterminePartitionsJobTest.java | 2 +- .../indexer/IndexGeneratorCombinerTest.java | 2 +- .../druid/indexer/IndexGeneratorJobTest.java | 2 +- .../io/druid/indexer/InputRowSerdeTest.java | 6 +- .../java/io/druid/indexer/JobHelperTest.java | 2 +- .../indexer/path/DatasourcePathSpecTest.java | 2 +- .../updater/HadoopConverterJobTest.java | 2 +- .../indexing/common/task/IndexTaskTest.java | 6 +- .../indexing/common/task/TaskSerdeTest.java | 4 +- .../IngestSegmentFirehoseFactoryTest.java | 4 +- ...estSegmentFirehoseFactoryTimelineTest.java | 2 +- .../indexing/overlord/TaskLifecycleTest.java | 8 +- .../aggregation/CountAggregatorFactory.java | 2 +- .../aggregation/DoubleSumAggregator.java | 8 +- .../DoubleSumAggregatorFactory.java | 15 +- .../DoubleSumBufferAggregator.java | 10 +- .../query/aggregation/LongSumAggregator.java | 10 +- .../aggregation/LongSumAggregatorFactory.java | 15 +- .../aggregation/LongSumBufferAggregator.java | 11 +- .../java/io/druid/query/DataSourceTest.java | 2 +- .../test/java/io/druid/query/QueriesTest.java | 16 +- .../io/druid/query/QueryRunnerTestHelper.java | 6 +- .../io/druid/query/RetryQueryRunnerTest.java | 3 +- .../query/aggregation/AggregatorUtilTest.java | 8 +- .../aggregation/DoubleSumAggregationTest.java | 145 ++++++++++++++++++ .../aggregation/DoubleSumAggregatorTest.java | 6 +- .../aggregation/FilteredAggregatorTest.java | 8 +- .../JavaScriptAggregatorBenchmark.java | 2 +- .../aggregation/JavaScriptAggregatorTest.java | 2 +- .../aggregation/LongSumAggregationTest.java | 144 +++++++++++++++++ .../aggregation/LongSumAggregatorTest.java | 6 +- .../aggregation/MetricManipulatorFnsTest.java | 5 +- .../query/groupby/GroupByQueryRunnerTest.java | 118 +++++++------- .../druid/query/groupby/GroupByQueryTest.java | 2 +- .../TimeSeriesUnionQueryRunnerTest.java | 6 +- .../timeseries/TimeseriesBinaryFnTest.java | 2 +- .../timeseries/TimeseriesQueryRunnerTest.java | 30 ++-- .../query/topn/TopNBinaryFnBenchmark.java | 2 +- .../io/druid/query/topn/TopNBinaryFnTest.java | 2 +- .../java/io/druid/segment/AppendTest.java | 6 +- .../io/druid/segment/IndexMergerTest.java | 10 +- .../io/druid/segment/SchemalessIndex.java | 4 +- .../io/druid/segment/SchemalessTestFull.java | 2 +- .../druid/segment/SchemalessTestSimple.java | 2 +- .../test/java/io/druid/segment/TestIndex.java | 2 +- .../segment/data/IncrementalIndexTest.java | 12 +- .../filter/SpatialFilterBonusTest.java | 6 +- .../segment/filter/SpatialFilterTest.java | 8 +- .../IncrementalIndexStorageAdapterTest.java | 9 +- .../OnheapIncrementalIndexBenchmark.java | 12 +- .../client/CachingClusteredClientTest.java | 8 +- .../druid/client/CachingQueryRunnerTest.java | 4 +- .../segment/indexing/DataSchemaTest.java | 14 +- .../firehose/IngestSegmentFirehoseTest.java | 2 +- ...criptTieredBrokerSelectorStrategyTest.java | 8 +- 57 files changed, 543 insertions(+), 198 deletions(-) create mode 100644 processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregationTest.java create mode 100644 processing/src/test/java/io/druid/query/aggregation/LongSumAggregationTest.java diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/BatchDeltaIngestionTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/BatchDeltaIngestionTest.java index 3350041abe54..eb52f7c7c4b7 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/BatchDeltaIngestionTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/BatchDeltaIngestionTest.java @@ -354,7 +354,7 @@ private HadoopDruidIndexerConfig makeHadoopDruidIndexerConfig(Map constructorFeeder() throws IOException .withDimensionsSpec(ROW_PARSER) .withMetrics( new AggregatorFactory[]{ - new LongSumAggregatorFactory(METRIC_LONG_NAME, DIM_LONG_NAME), - new DoubleSumAggregatorFactory(METRIC_FLOAT_NAME, DIM_FLOAT_NAME) + new LongSumAggregatorFactory(METRIC_LONG_NAME, DIM_LONG_NAME, 1), + new DoubleSumAggregatorFactory(METRIC_FLOAT_NAME, DIM_FLOAT_NAME, 1) } ) .build(); diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java index 3bd5de2c6ae0..b621d0efe8e3 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java @@ -209,7 +209,7 @@ private static Map persist(File tmpDir, InputRow... rows) .withDimensionsSpec(ROW_PARSER) .withMetrics( new AggregatorFactory[]{ - new LongSumAggregatorFactory(METRICS[0], METRICS[0]) + new LongSumAggregatorFactory(METRICS[0], METRICS[0], 1) } ) .build(); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java index 7d326d765677..f9728f9764a5 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java @@ -507,7 +507,7 @@ public void testIndexTask() throws Exception new DataSchema( "foo", null, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, + new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met", 1)}, new UniformGranularitySpec( Granularity.DAY, null, @@ -565,7 +565,7 @@ public void testIndexTaskFailure() throws Exception new DataSchema( "foo", null, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, + new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met", 1)}, new UniformGranularitySpec( Granularity.DAY, null, @@ -914,7 +914,7 @@ public void testResumeTasks() throws Exception new DataSchema( "foo", null, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, + new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met", 1)}, new UniformGranularitySpec( Granularity.DAY, null, @@ -1012,7 +1012,7 @@ private RealtimeIndexTask giveMeARealtimeIndexTask() { DataSchema dataSchema = new DataSchema( "test_ds", null, - new AggregatorFactory[]{new LongSumAggregatorFactory("count", "rows")}, + new AggregatorFactory[]{new LongSumAggregatorFactory("count", "rows", 1)}, new UniformGranularitySpec(Granularity.DAY, QueryGranularity.NONE, null), mapper ); diff --git a/processing/src/main/java/io/druid/query/aggregation/CountAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/CountAggregatorFactory.java index ae4243d5e4ff..5ff99b90cd1f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/CountAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/CountAggregatorFactory.java @@ -72,7 +72,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongSumAggregatorFactory(name, name); + return new LongSumAggregatorFactory(name, name, 1); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregator.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregator.java index b6972b8a0a65..db0f88c647d2 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregator.java @@ -43,13 +43,15 @@ static double combineValues(Object lhs, Object rhs) private final FloatColumnSelector selector; private final String name; + private final int exponent; private double sum; - public DoubleSumAggregator(String name, FloatColumnSelector selector) + public DoubleSumAggregator(String name, FloatColumnSelector selector, int exponent) { this.name = name; this.selector = selector; + this.exponent = exponent; this.sum = 0; } @@ -57,7 +59,7 @@ public DoubleSumAggregator(String name, FloatColumnSelector selector) @Override public void aggregate() { - sum += selector.get(); + sum += (exponent == 1 ? selector.get() : Math.pow(selector.get(), exponent)); } @Override @@ -93,7 +95,7 @@ public String getName() @Override public Aggregator clone() { - return new DoubleSumAggregator(name, selector); + return new DoubleSumAggregator(name, selector, exponent); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 87f98b223b19..e62c551a4459 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -37,11 +37,13 @@ public class DoubleSumAggregatorFactory implements AggregatorFactory private final String fieldName; private final String name; + private final int exponent; @JsonCreator public DoubleSumAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("exponent") final Integer exponent ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); @@ -49,6 +51,8 @@ public DoubleSumAggregatorFactory( this.name = name; this.fieldName = fieldName; + this.exponent = exponent == null ? 1 : exponent.intValue(); + Preconditions.checkArgument(this.exponent >= 1, "exponent must be greater or equal to 1"); } @Override @@ -56,14 +60,15 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) { return new DoubleSumAggregator( name, - metricFactory.makeFloatColumnSelector(fieldName) + metricFactory.makeFloatColumnSelector(fieldName), + exponent ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleSumBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); + return new DoubleSumBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName), exponent); } @Override @@ -81,13 +86,13 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new DoubleSumAggregatorFactory(name, name); + return new DoubleSumAggregatorFactory(name, name, 1); } @Override public List getRequiredColumns() { - return Arrays.asList(new DoubleSumAggregatorFactory(fieldName, fieldName)); + return Arrays.asList(new DoubleSumAggregatorFactory(fieldName, fieldName, exponent)); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java index b57dcad3eb4f..a3e60247b0f6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java @@ -26,12 +26,15 @@ public class DoubleSumBufferAggregator implements BufferAggregator { private final FloatColumnSelector selector; + private final int exponent; public DoubleSumBufferAggregator( - FloatColumnSelector selector + FloatColumnSelector selector, + int exponent ) { this.selector = selector; + this.exponent = exponent; } @Override @@ -43,7 +46,10 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position) { - buf.putDouble(position, buf.getDouble(position) + (double) selector.get()); + buf.putDouble( + position, + buf.getDouble(position) + (exponent == 1 ? + selector.get() : Math.pow(selector.get(), exponent)) + ); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregator.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregator.java index c8547ead2d96..0522b172d860 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregator.java @@ -17,6 +17,7 @@ package io.druid.query.aggregation; +import com.google.common.math.LongMath; import com.google.common.primitives.Longs; import io.druid.segment.LongColumnSelector; @@ -41,13 +42,15 @@ static long combineValues(Object lhs, Object rhs) { private final LongColumnSelector selector; private final String name; + private final int exponent; private long sum; - public LongSumAggregator(String name, LongColumnSelector selector) + public LongSumAggregator(String name, LongColumnSelector selector, int exponent) { this.name = name; this.selector = selector; + this.exponent = exponent; this.sum = 0; } @@ -55,7 +58,8 @@ public LongSumAggregator(String name, LongColumnSelector selector) @Override public void aggregate() { - sum += selector.get(); + //TODO: should we do LongMath.checkPow(..) instead which would fail in case of overflow? + sum += (exponent == 1 ? selector.get() : LongMath.pow(selector.get(), exponent)); } @Override @@ -91,7 +95,7 @@ public String getName() @Override public Aggregator clone() { - return new LongSumAggregator(name, selector); + return new LongSumAggregator(name, selector, exponent); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 48a598d9cf6a..a0998c157005 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -37,11 +37,13 @@ public class LongSumAggregatorFactory implements AggregatorFactory private final String fieldName; private final String name; + private final int exponent; @JsonCreator public LongSumAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("exponent") final Integer exponent ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); @@ -49,6 +51,8 @@ public LongSumAggregatorFactory( this.name = name; this.fieldName = fieldName; + this.exponent = exponent == null ? 1 : exponent.intValue(); + Preconditions.checkArgument(this.exponent >= 1, "exponent must be greater or equal to 1"); } @Override @@ -56,14 +60,15 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) { return new LongSumAggregator( name, - metricFactory.makeLongColumnSelector(fieldName) + metricFactory.makeLongColumnSelector(fieldName), + exponent ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongSumBufferAggregator(metricFactory.makeLongColumnSelector(fieldName)); + return new LongSumBufferAggregator(metricFactory.makeLongColumnSelector(fieldName), exponent); } @Override @@ -81,13 +86,13 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongSumAggregatorFactory(name, name); + return new LongSumAggregatorFactory(name, name, 1); } @Override public List getRequiredColumns() { - return Arrays.asList(new LongSumAggregatorFactory(fieldName, fieldName)); + return Arrays.asList(new LongSumAggregatorFactory(fieldName, fieldName, exponent)); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/LongSumBufferAggregator.java index 3e2f0aaf5fa8..d4ce1dded49f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumBufferAggregator.java @@ -17,6 +17,7 @@ package io.druid.query.aggregation; +import com.google.common.math.LongMath; import io.druid.segment.LongColumnSelector; import java.nio.ByteBuffer; @@ -26,12 +27,15 @@ public class LongSumBufferAggregator implements BufferAggregator { private final LongColumnSelector selector; + private final int exponent; public LongSumBufferAggregator( - LongColumnSelector selector + LongColumnSelector selector, + int exponent ) { this.selector = selector; + this.exponent = exponent; } @Override @@ -43,7 +47,10 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position) { - buf.putLong(position, buf.getLong(position) + selector.get()); + buf.putLong( + position, + buf.getLong(position) + (exponent == 1 ? + selector.get() : LongMath.pow(selector.get(), exponent)) + ); } @Override diff --git a/processing/src/test/java/io/druid/query/DataSourceTest.java b/processing/src/test/java/io/druid/query/DataSourceTest.java index b8f597b5e0e5..75161eb2b484 100644 --- a/processing/src/test/java/io/druid/query/DataSourceTest.java +++ b/processing/src/test/java/io/druid/query/DataSourceTest.java @@ -69,7 +69,7 @@ public void testQueryDataSource() throws IOException .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) diff --git a/processing/src/test/java/io/druid/query/QueriesTest.java b/processing/src/test/java/io/druid/query/QueriesTest.java index 1ce604066c9e..556b341b0a76 100644 --- a/processing/src/test/java/io/druid/query/QueriesTest.java +++ b/processing/src/test/java/io/druid/query/QueriesTest.java @@ -39,8 +39,8 @@ public void testVerifyAggregations() throws Exception { List aggFactories = Arrays.asList( new CountAggregatorFactory("count"), - new DoubleSumAggregatorFactory("idx", "index"), - new DoubleSumAggregatorFactory("rev", "revenue") + new DoubleSumAggregatorFactory("idx", "index", 1), + new DoubleSumAggregatorFactory("rev", "revenue", 1) ); List postAggs = Arrays.asList( @@ -71,8 +71,8 @@ public void testVerifyAggregationsMissingVal() throws Exception { List aggFactories = Arrays.asList( new CountAggregatorFactory("count"), - new DoubleSumAggregatorFactory("idx", "index"), - new DoubleSumAggregatorFactory("rev", "revenue") + new DoubleSumAggregatorFactory("idx", "index", 1), + new DoubleSumAggregatorFactory("rev", "revenue", 1) ); List postAggs = Arrays.asList( @@ -103,8 +103,8 @@ public void testVerifyAggregationsMultiLevel() throws Exception { List aggFactories = Arrays.asList( new CountAggregatorFactory("count"), - new DoubleSumAggregatorFactory("idx", "index"), - new DoubleSumAggregatorFactory("rev", "revenue") + new DoubleSumAggregatorFactory("idx", "index", 1), + new DoubleSumAggregatorFactory("rev", "revenue", 1) ); List postAggs = Arrays.asList( @@ -157,8 +157,8 @@ public void testVerifyAggregationsMultiLevelMissingVal() throws Exception { List aggFactories = Arrays.asList( new CountAggregatorFactory("count"), - new DoubleSumAggregatorFactory("idx", "index"), - new DoubleSumAggregatorFactory("rev", "revenue") + new DoubleSumAggregatorFactory("idx", "index", 1), + new DoubleSumAggregatorFactory("rev", "revenue", 1) ); List postAggs = Arrays.asList( diff --git a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index 318aeedd3302..2cd41fcdf2df 100644 --- a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -98,9 +98,9 @@ public TableDataSource apply(@Nullable String input) public static final String addRowsIndexConstantMetric = "addRowsIndexConstant"; public static String dependentPostAggMetric = "dependentPostAgg"; public static final CountAggregatorFactory rowsCount = new CountAggregatorFactory("rows"); - public static final LongSumAggregatorFactory indexLongSum = new LongSumAggregatorFactory("index", "index"); - public static final LongSumAggregatorFactory __timeLongSum = new LongSumAggregatorFactory("sumtime", "__time"); - public static final DoubleSumAggregatorFactory indexDoubleSum = new DoubleSumAggregatorFactory("index", "index"); + public static final LongSumAggregatorFactory indexLongSum = new LongSumAggregatorFactory("index", "index", 1); + public static final LongSumAggregatorFactory __timeLongSum = new LongSumAggregatorFactory("sumtime", "__time", 1); + public static final DoubleSumAggregatorFactory indexDoubleSum = new DoubleSumAggregatorFactory("index", "index", 1); public static final String JS_COMBINE_A_PLUS_B = "function combine(a, b) { return a + b; }"; public static final String JS_RESET_0 = "function reset() { return 0; }"; public static final JavaScriptAggregatorFactory jsIndexSumIfPlacementishA = new JavaScriptAggregatorFactory( diff --git a/processing/src/test/java/io/druid/query/RetryQueryRunnerTest.java b/processing/src/test/java/io/druid/query/RetryQueryRunnerTest.java index a1c9ad2e9be7..0ffeb83ea73b 100644 --- a/processing/src/test/java/io/druid/query/RetryQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/RetryQueryRunnerTest.java @@ -51,7 +51,8 @@ public class RetryQueryRunnerTest QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ), QueryRunnerTestHelper.qualityUniques ) diff --git a/processing/src/test/java/io/druid/query/aggregation/AggregatorUtilTest.java b/processing/src/test/java/io/druid/query/aggregation/AggregatorUtilTest.java index 93a0c63e1c2e..f54d5c629b87 100644 --- a/processing/src/test/java/io/druid/query/aggregation/AggregatorUtilTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/AggregatorUtilTest.java @@ -146,8 +146,8 @@ public void testCondenseAggregators() @Test public void testNullPostAggregatorNames() { - AggregatorFactory agg1 = new DoubleSumAggregatorFactory("agg1", "value"); - AggregatorFactory agg2 = new DoubleSumAggregatorFactory("agg2", "count"); + AggregatorFactory agg1 = new DoubleSumAggregatorFactory("agg1", "value", 1); + AggregatorFactory agg2 = new DoubleSumAggregatorFactory("agg2", "count", 1); PostAggregator postAgg1 = new ArithmeticPostAggregator( null, "*", Lists.newArrayList( new FieldAccessPostAggregator( @@ -181,8 +181,8 @@ public void testNullPostAggregatorNames() @Test public void testCasing() { - AggregatorFactory agg1 = new DoubleSumAggregatorFactory("Agg1", "value"); - AggregatorFactory agg2 = new DoubleSumAggregatorFactory("Agg2", "count"); + AggregatorFactory agg1 = new DoubleSumAggregatorFactory("Agg1", "value", 1); + AggregatorFactory agg2 = new DoubleSumAggregatorFactory("Agg2", "count", 1); PostAggregator postAgg1 = new ArithmeticPostAggregator( null, "*", Lists.newArrayList( new FieldAccessPostAggregator( diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregationTest.java new file mode 100644 index 000000000000..fbf9af320791 --- /dev/null +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregationTest.java @@ -0,0 +1,145 @@ +/* +* Licensed to Metamarkets Group Inc. (Metamarkets) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. Metamarkets licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package io.druid.query.aggregation; + +import com.google.common.collect.Lists; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import io.druid.data.input.MapBasedRow; +import io.druid.granularity.QueryGranularity; +import io.druid.jackson.AggregatorsModule; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +/** + */ +public class DoubleSumAggregationTest +{ + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + private File dataFile; + private String parseSpec; + private String query; + + @Before + public void setup() throws Exception + { + String line1 = "2011-01-12T00:00:00.000Z\tproduct1\t2.1"; + String line2 = "2011-01-14T00:00:00.000Z\tproduct2\t3.2"; + + List lines = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + lines.add(line1); + lines.add(line2); + } + + this.dataFile = tempFolder.newFile(); + FileUtils.writeLines(dataFile, "UTF-8", lines, "\n"); + + this.parseSpec = "{" + + "\"type\" : \"string\"," + + "\"parseSpec\" : {" + + " \"format\" : \"tsv\"," + + " \"timestampSpec\" : {" + + " \"column\" : \"timestamp\"," + + " \"format\" : \"auto\"" + + "}," + + " \"dimensionsSpec\" : {" + + " \"dimensions\": [\"product\"]," + + " \"dimensionExclusions\" : []," + + " \"spatialDimensions\" : []" + + " }," + + " \"columns\": [\"timestamp\", \"product\", \"quantity\"]" + + " }" + + "}"; + + this.query = "{" + + "\"queryType\": \"groupBy\"," + + "\"dataSource\": \"test_datasource\"," + + "\"granularity\": \"ALL\"," + + "\"dimensions\": []," + + "\"aggregations\": [" + + " { \"type\": \"doubleSum\", \"name\": \"quantity\", \"fieldName\": \"quantity\" }" + + "]," + + "\"intervals\": [ \"1970/2050\" ]" + + "}"; + } + + @Test + public void testIngestAndQueryNullExponent() throws Exception + { + AggregationTestHelper helper = new AggregationTestHelper(Lists.newArrayList(new AggregatorsModule()), tempFolder); + + String metricSpec = "[{" + + "\"type\": \"doubleSum\"," + + "\"name\": \"quantity\"," + + "\"fieldName\": \"quantity\"" + + "}]"; + + Sequence seq = helper.createIndexAndRunQueryOnSegment( + dataFile, + parseSpec, + metricSpec, + 0, + QueryGranularity.NONE, + 50000, + query + ); + + MapBasedRow row = (MapBasedRow) Sequences.toList(seq, Lists.newArrayList()).get(0); + Assert.assertEquals(26.5, row.getFloatMetric("quantity"), 0.001); + } + + @Test + public void testIngestAndQuerySquareExponent() throws Exception + { + AggregationTestHelper helper = new AggregationTestHelper(Lists.newArrayList(new AggregatorsModule()), tempFolder); + + String metricSpec = "[{" + + "\"type\": \"doubleSum\"," + + "\"name\": \"quantity\"," + + "\"fieldName\": \"quantity\"," + + "\"exponent\": \"2\"" + + "}]"; + + Sequence seq = helper.createIndexAndRunQueryOnSegment( + dataFile, + parseSpec, + metricSpec, + 0, + QueryGranularity.NONE, + 50000, + query + ); + + MapBasedRow row = (MapBasedRow) Sequences.toList(seq, Lists.newArrayList()).get(0); + Assert.assertEquals(73.25, row.getFloatMetric("quantity"), 0.001); + } +} + diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregatorTest.java index 1c9f24f3a239..b72c71a83377 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregatorTest.java @@ -37,7 +37,7 @@ public void testAggregate() { final float[] values = {0.15f, 0.27f}; final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); - DoubleSumAggregator agg = new DoubleSumAggregator("billy", selector); + DoubleSumAggregator agg = new DoubleSumAggregator("billy", selector, 1); Assert.assertEquals("billy", agg.getName()); @@ -61,14 +61,14 @@ public void testAggregate() public void testComparator() { final TestFloatColumnSelector selector = new TestFloatColumnSelector(new float[]{0.15f, 0.27f}); - DoubleSumAggregator agg = new DoubleSumAggregator("billy", selector); + DoubleSumAggregator agg = new DoubleSumAggregator("billy", selector, 1); Assert.assertEquals("billy", agg.getName()); Object first = agg.get(); agg.aggregate(); - Comparator comp = new DoubleSumAggregatorFactory("null", "null").getComparator(); + Comparator comp = new DoubleSumAggregatorFactory("null", "null", 1).getComparator(); Assert.assertEquals(-1, comp.compare(first, agg.get())); Assert.assertEquals(0, comp.compare(first, first)); diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 603dec6c057f..f71139509cd5 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -49,7 +49,7 @@ public void testAggregate() final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); FilteredAggregatorFactory factory = new FilteredAggregatorFactory( - new DoubleSumAggregatorFactory("billy", "value"), + new DoubleSumAggregatorFactory("billy", "value", 1), new SelectorDimFilter("dim", "a") ); @@ -166,7 +166,7 @@ public void testAggregateWithNotFilter() final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); FilteredAggregatorFactory factory = new FilteredAggregatorFactory( - new DoubleSumAggregatorFactory("billy", "value"), + new DoubleSumAggregatorFactory("billy", "value", 1), new NotDimFilter(new SelectorDimFilter("dim", "b")) ); @@ -189,7 +189,7 @@ public void testAggregateWithOrFilter() final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); FilteredAggregatorFactory factory = new FilteredAggregatorFactory( - new DoubleSumAggregatorFactory("billy", "value"), + new DoubleSumAggregatorFactory("billy", "value", 1), new OrDimFilter(Lists.newArrayList(new SelectorDimFilter("dim", "a"), new SelectorDimFilter("dim", "b"))) ); @@ -212,7 +212,7 @@ public void testAggregateWithAndFilter() final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); FilteredAggregatorFactory factory = new FilteredAggregatorFactory( - new DoubleSumAggregatorFactory("billy", "value"), + new DoubleSumAggregatorFactory("billy", "value", 1), new AndDimFilter(Lists.newArrayList(new NotDimFilter(new SelectorDimFilter("dim", "b")), new SelectorDimFilter("dim", "a")))); FilteredAggregator agg = (FilteredAggregator) factory.factorize( diff --git a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorBenchmark.java b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorBenchmark.java index 02bad9fcd3ca..88214c0081b6 100644 --- a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorBenchmark.java +++ b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorBenchmark.java @@ -60,7 +60,7 @@ protected void setUp() throws Exception ) ); - doubleAgg = new DoubleSumAggregator("billy", selector); + doubleAgg = new DoubleSumAggregator("billy", selector, 1); } public double timeJavaScriptDoubleSum(int reps) diff --git a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java index dd4e89c2d792..f70a8119ddce 100644 --- a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java @@ -249,7 +249,7 @@ public static void main(String... args) throws Exception { script.get("fnCombine")) ); - DoubleSumAggregator doubleAgg = new DoubleSumAggregator("billy", selector); + DoubleSumAggregator doubleAgg = new DoubleSumAggregator("billy", selector, 1); // warmup int i = 0; diff --git a/processing/src/test/java/io/druid/query/aggregation/LongSumAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongSumAggregationTest.java new file mode 100644 index 000000000000..1de49cf5c916 --- /dev/null +++ b/processing/src/test/java/io/druid/query/aggregation/LongSumAggregationTest.java @@ -0,0 +1,144 @@ +/* +* Licensed to Metamarkets Group Inc. (Metamarkets) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. Metamarkets licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package io.druid.query.aggregation; + +import com.google.common.collect.Lists; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import io.druid.data.input.MapBasedRow; +import io.druid.granularity.QueryGranularity; +import io.druid.jackson.AggregatorsModule; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +/** + */ +public class LongSumAggregationTest +{ + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + private File dataFile; + private String parseSpec; + private String query; + + @Before + public void setup() throws Exception + { + String line1 = "2011-01-12T00:00:00.000Z\tproduct1\t2"; + String line2 = "2011-01-14T00:00:00.000Z\tproduct2\t3"; + + List lines = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + lines.add(line1); + lines.add(line2); + } + + this.dataFile = tempFolder.newFile(); + FileUtils.writeLines(dataFile, "UTF-8", lines, "\n"); + + this.parseSpec = "{" + + "\"type\" : \"string\"," + + "\"parseSpec\" : {" + + " \"format\" : \"tsv\"," + + " \"timestampSpec\" : {" + + " \"column\" : \"timestamp\"," + + " \"format\" : \"auto\"" + + "}," + + " \"dimensionsSpec\" : {" + + " \"dimensions\": [\"product\"]," + + " \"dimensionExclusions\" : []," + + " \"spatialDimensions\" : []" + + " }," + + " \"columns\": [\"timestamp\", \"product\", \"quantity\"]" + + " }" + + "}"; + + this.query = "{" + + "\"queryType\": \"groupBy\"," + + "\"dataSource\": \"test_datasource\"," + + "\"granularity\": \"ALL\"," + + "\"dimensions\": []," + + "\"aggregations\": [" + + " { \"type\": \"longSum\", \"name\": \"quantity\", \"fieldName\": \"quantity\" }" + + "]," + + "\"intervals\": [ \"1970/2050\" ]" + + "}"; + } + + @Test + public void testIngestAndQueryNullExponent() throws Exception + { + AggregationTestHelper helper = new AggregationTestHelper(Lists.newArrayList(new AggregatorsModule()), tempFolder); + + String metricSpec = "[{" + + "\"type\": \"longSum\"," + + "\"name\": \"quantity\"," + + "\"fieldName\": \"quantity\"" + + "}]"; + + Sequence seq = helper.createIndexAndRunQueryOnSegment( + dataFile, + parseSpec, + metricSpec, + 0, + QueryGranularity.NONE, + 50000, + query + ); + + MapBasedRow row = (MapBasedRow) Sequences.toList(seq, Lists.newArrayList()).get(0); + Assert.assertEquals(25, row.getLongMetric("quantity")); + } + + @Test + public void testIngestAndQuerySquareExponent() throws Exception + { + AggregationTestHelper helper = new AggregationTestHelper(Lists.newArrayList(new AggregatorsModule()), tempFolder); + + String metricSpec = "[{" + + "\"type\": \"longSum\"," + + "\"name\": \"quantity\"," + + "\"fieldName\": \"quantity\"," + + "\"exponent\": \"2\"" + + "}]"; + + Sequence seq = helper.createIndexAndRunQueryOnSegment( + dataFile, + parseSpec, + metricSpec, + 0, + QueryGranularity.NONE, + 50000, + query + ); + + MapBasedRow row = (MapBasedRow) Sequences.toList(seq, Lists.newArrayList()).get(0); + Assert.assertEquals(65, row.getLongMetric("quantity")); + } +} diff --git a/processing/src/test/java/io/druid/query/aggregation/LongSumAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/LongSumAggregatorTest.java index 7e5074b98baf..b4a3cf63dc53 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongSumAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongSumAggregatorTest.java @@ -36,7 +36,7 @@ private void aggregate(TestLongColumnSelector selector, LongSumAggregator agg) public void testAggregate() { final TestLongColumnSelector selector = new TestLongColumnSelector(new long[]{24L, 20L}); - LongSumAggregator agg = new LongSumAggregator("billy", selector); + LongSumAggregator agg = new LongSumAggregator("billy", selector, 1); Assert.assertEquals("billy", agg.getName()); @@ -57,14 +57,14 @@ public void testAggregate() public void testComparator() { final TestLongColumnSelector selector = new TestLongColumnSelector(new long[]{18293L}); - LongSumAggregator agg = new LongSumAggregator("billy", selector); + LongSumAggregator agg = new LongSumAggregator("billy", selector, 1); Assert.assertEquals("billy", agg.getName()); Object first = agg.get(); agg.aggregate(); - Comparator comp = new LongSumAggregatorFactory("null", "null").getComparator(); + Comparator comp = new LongSumAggregatorFactory("null", "null", 1).getComparator(); Assert.assertEquals(-1, comp.compare(first, agg.get())); Assert.assertEquals(0, comp.compare(first, first)); diff --git a/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java b/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java index 490ddd89d0bc..c47cba9b0106 100644 --- a/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java @@ -79,7 +79,7 @@ public long get() ); - LongSumAggregatorFactory longSumAggregatorFactory = new LongSumAggregatorFactory(NAME, FIELD); + LongSumAggregatorFactory longSumAggregatorFactory = new LongSumAggregatorFactory(NAME, FIELD, 1); LongSumAggregator longSumAggregator = new LongSumAggregator( NAME, new LongColumnSelector() { @@ -88,7 +88,8 @@ public long get() { return longVal; } - } + }, + 1 ); constructorArrays.add( new Object[]{ diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 725037bd40a1..88509c939585 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -214,7 +214,7 @@ public void testGroupBy() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -273,7 +273,7 @@ public void testGroupByWithRebucketRename() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -349,7 +349,7 @@ public void testGroupByWithSimpleRenameRetainMissingNonInjective() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -425,7 +425,7 @@ public void testGroupByWithSimpleRenameRetainMissing() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -504,7 +504,7 @@ public void testGroupByWithSimpleRenameAndMissingString() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -579,7 +579,7 @@ public void testGroupByWithSimpleRename() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -749,7 +749,7 @@ public String apply(String dimValue) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -817,7 +817,7 @@ public String apply(String dimValue) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -876,7 +876,8 @@ public void testGroupByWithTimeZone() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ) ) ) @@ -1070,7 +1071,7 @@ public void testMergeResults() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); @@ -1148,7 +1149,7 @@ private void doTestMergeResultsWithValidLimit(final int limit) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) @@ -1188,7 +1189,7 @@ public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryGranularity.DAY) @@ -1235,7 +1236,7 @@ public void testMergeResultsWithNegativeLimit() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) @@ -1314,7 +1315,7 @@ private void doTestMergeResultsWithOrderBy(LimitSpec orderBySpec, List expe .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) @@ -1357,7 +1358,7 @@ public void testGroupByOrderLimit() throws Exception .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .addOrderByColumn("rows") @@ -1398,7 +1399,7 @@ public void testGroupByWithOrderLimit2() throws Exception .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .addOrderByColumn("rows", "desc") @@ -1438,7 +1439,7 @@ public void testGroupByWithOrderLimit3() throws Exception .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new DoubleSumAggregatorFactory("idx", "index") + new DoubleSumAggregatorFactory("idx", "index", 1) ) ) .addOrderByColumn("idx", "desc") @@ -1738,7 +1739,7 @@ public void testPostAggMergedHavingSpec() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("index", "index") + new LongSumAggregatorFactory("index", "index", 1) ) ) .setPostAggregatorSpecs(ImmutableList.of(QueryRunnerTestHelper.addRowsIndexConstant)) @@ -1788,7 +1789,7 @@ public void testGroupByWithOrderLimitHavingSpec() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new DoubleSumAggregatorFactory("index", "index") + new DoubleSumAggregatorFactory("index", "index", 1) ) ) .setGranularity(QueryGranularity.ALL) @@ -1898,7 +1899,7 @@ public void testPostAggHavingSpec() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("index", "index") + new LongSumAggregatorFactory("index", "index", 1) ) ) .setPostAggregatorSpecs(ImmutableList.of(QueryRunnerTestHelper.addRowsIndexConstant)) @@ -1937,7 +1938,7 @@ public void testHavingSpec() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) @@ -1975,7 +1976,7 @@ public void testMergedHavingSpec() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) @@ -2061,7 +2062,7 @@ public void testMergedPostAggHavingSpec() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setPostAggregatorSpecs( @@ -2350,7 +2351,7 @@ public void testIdenticalSubquery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -2363,8 +2364,8 @@ public void testIdenticalSubquery() .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("alias", "alias"))) .setAggregatorSpecs( Arrays.asList( - new LongSumAggregatorFactory("rows", "rows"), - new LongSumAggregatorFactory("idx", "idx") + new LongSumAggregatorFactory("rows", "rows", 1), + new LongSumAggregatorFactory("idx", "idx", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -2409,7 +2410,7 @@ public void testSubqueryWithMultipleIntervalsInOuterQuery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -2429,8 +2430,8 @@ public void testSubqueryWithMultipleIntervalsInOuterQuery() .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("alias", "alias"))) .setAggregatorSpecs( Arrays.asList( - new LongSumAggregatorFactory("rows", "rows"), - new LongSumAggregatorFactory("idx", "idx") + new LongSumAggregatorFactory("rows", "rows", 1), + new LongSumAggregatorFactory("idx", "idx", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -2474,7 +2475,7 @@ public void testDifferentGroupingSubquery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -2512,7 +2513,7 @@ public void testDifferentGroupingSubqueryMultipleAggregatorsOnSameField() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setPostAggregatorSpecs( @@ -2568,7 +2569,7 @@ public void testDifferentGroupingSubqueryWithFilter() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -2621,7 +2622,7 @@ public void testDifferentIntervalSubquery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -2658,7 +2659,7 @@ public void testEmptySubquery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -2692,7 +2693,7 @@ public void testSubqueryWithPostAggregators() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx_subagg", "index") + new LongSumAggregatorFactory("idx_subagg", "index", 1) ) ) .setPostAggregatorSpecs( @@ -2716,8 +2717,8 @@ public void testSubqueryWithPostAggregators() .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("alias", "alias"))) .setAggregatorSpecs( Arrays.asList( - new LongSumAggregatorFactory("rows", "rows"), - new LongSumAggregatorFactory("idx", "idx_subpostagg") + new LongSumAggregatorFactory("rows", "rows", 1), + new LongSumAggregatorFactory("idx", "idx_subpostagg", 1) ) ) .setPostAggregatorSpecs( @@ -2953,7 +2954,7 @@ public void testSubqueryWithPostAggregatorsAndHaving() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx_subagg", "index") + new LongSumAggregatorFactory("idx_subagg", "index", 1) ) ) .setPostAggregatorSpecs( @@ -2996,8 +2997,8 @@ public byte[] getCacheKey() .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("alias", "alias"))) .setAggregatorSpecs( Arrays.asList( - new LongSumAggregatorFactory("rows", "rows"), - new LongSumAggregatorFactory("idx", "idx_subpostagg") + new LongSumAggregatorFactory("rows", "rows", 1), + new LongSumAggregatorFactory("idx", "idx_subpostagg", 1) ) ) .setPostAggregatorSpecs( @@ -3211,7 +3212,7 @@ public void testSubqueryWithMultiColumnAggregators() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new DoubleSumAggregatorFactory("idx_subagg", "index"), + new DoubleSumAggregatorFactory("idx_subagg", "index", 1), new JavaScriptAggregatorFactory( "js_agg", Arrays.asList("index", "market"), @@ -3261,9 +3262,9 @@ public byte[] getCacheKey() .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("alias", "alias"))) .setAggregatorSpecs( Arrays.asList( - new LongSumAggregatorFactory("rows", "rows"), - new LongSumAggregatorFactory("idx", "idx_subpostagg"), - new DoubleSumAggregatorFactory("js_outer_agg", "js_agg") + new LongSumAggregatorFactory("rows", "rows", 1), + new LongSumAggregatorFactory("idx", "idx_subpostagg", 1), + new DoubleSumAggregatorFactory("js_outer_agg", "js_agg", 1) ) ) .setPostAggregatorSpecs( @@ -3375,7 +3376,7 @@ public void testSubqueryWithHyperUniques() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index"), + new LongSumAggregatorFactory("idx", "index", 1), new HyperUniquesAggregatorFactory("quality_uniques", "quality_uniques") ) ) @@ -3389,8 +3390,8 @@ public void testSubqueryWithHyperUniques() .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("alias", "alias"))) .setAggregatorSpecs( Arrays.asList( - new LongSumAggregatorFactory("rows", "rows"), - new LongSumAggregatorFactory("idx", "idx"), + new LongSumAggregatorFactory("rows", "rows", 1), + new LongSumAggregatorFactory("idx", "idx", 1), new HyperUniquesAggregatorFactory("uniq", "quality_uniques") ) ) @@ -3515,7 +3516,7 @@ public void testSubqueryWithHyperUniquesPostAggregator() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index"), + new LongSumAggregatorFactory("idx", "index", 1), new HyperUniquesAggregatorFactory("quality_uniques_inner", "quality_uniques") ) ) @@ -3534,8 +3535,8 @@ public void testSubqueryWithHyperUniquesPostAggregator() .setDimensions(Lists.newArrayList()) .setAggregatorSpecs( Arrays.asList( - new LongSumAggregatorFactory("rows", "rows"), - new LongSumAggregatorFactory("idx", "idx"), + new LongSumAggregatorFactory("rows", "rows", 1), + new LongSumAggregatorFactory("idx", "idx", 1), new HyperUniquesAggregatorFactory("quality_uniques_outer", "quality_uniques_inner_post") ) ) @@ -3856,7 +3857,7 @@ public void testBySegmentResults() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) @@ -3930,7 +3931,7 @@ public void testBySegmentResultsUnOptimizedDimextraction() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) @@ -4003,7 +4004,7 @@ public void testBySegmentResultsOptimizedDimextraction() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) @@ -4064,7 +4065,7 @@ public void testGroupByWithExtractionDimFilter() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -4126,7 +4127,7 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -4163,7 +4164,7 @@ public void testGroupByWithExtractionDimFilterWhenSearchValueNotInTheMap() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -4204,7 +4205,7 @@ public void testGroupByWithExtractionDimFilterKeyisNull() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -4259,7 +4260,8 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() (AggregatorFactory) new FilteredAggregatorFactory( new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ), filter ) ) diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryTest.java index 4203164777c2..6479aba2321b 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryTest.java @@ -47,7 +47,7 @@ public void testQuerySerialization() throws IOException .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index", 1) ) ) .setGranularity(QueryRunnerTestHelper.dayGran) diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java index 68afc07bda13..9d93727ee8aa 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java @@ -83,7 +83,8 @@ public void testUnionTimeseries() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ), QueryRunnerTestHelper.qualityUniques ) @@ -132,7 +133,8 @@ public void testUnionResultMerging() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ) ) ) diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesBinaryFnTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesBinaryFnTest.java index 991ce2375abb..82f55c6c884e 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesBinaryFnTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesBinaryFnTest.java @@ -35,7 +35,7 @@ public class TimeseriesBinaryFnTest { final CountAggregatorFactory rowsCount = new CountAggregatorFactory("rows"); - final LongSumAggregatorFactory indexLongSum = new LongSumAggregatorFactory("index", "index"); + final LongSumAggregatorFactory indexLongSum = new LongSumAggregatorFactory("index", "index", 1); final List aggregatorFactories = Arrays.asList( rowsCount, indexLongSum diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index 2b18b8eb1f29..1e6b0674c2fb 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -268,7 +268,8 @@ public void testTimeseries() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ), QueryRunnerTestHelper.qualityUniques ) @@ -309,7 +310,8 @@ public void testTimeseriesWithTimeZone() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ) ) ) @@ -363,7 +365,8 @@ public void testTimeseriesWithVaryingGran() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ), QueryRunnerTestHelper.qualityUniques ) @@ -400,7 +403,8 @@ public void testTimeseriesWithVaryingGran() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ), QueryRunnerTestHelper.qualityUniques ) @@ -448,7 +452,8 @@ public void testTimeseriesGranularityNotAlignedOnSegmentBoundariesWithFilter() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ) ) ) @@ -495,7 +500,8 @@ public void testTimeseriesQueryZeroFilling() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ) ) ) @@ -569,7 +575,8 @@ public void testTimeseriesQueryGranularityNotAlignedWithRollupGranularity() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ) ) ) @@ -610,7 +617,8 @@ public void testTimeseriesWithVaryingGranWithFilter() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ), QueryRunnerTestHelper.qualityUniques ) @@ -647,7 +655,8 @@ public void testTimeseriesWithVaryingGranWithFilter() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ), QueryRunnerTestHelper.qualityUniques ) @@ -690,7 +699,8 @@ public void testTimeseriesQueryBeyondTimeRangeOfData() QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory( "idx", - "index" + "index", + 1 ) ) ) diff --git a/processing/src/test/java/io/druid/query/topn/TopNBinaryFnBenchmark.java b/processing/src/test/java/io/druid/query/topn/TopNBinaryFnBenchmark.java index 0e3785b120cd..99a768c1c61e 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNBinaryFnBenchmark.java +++ b/processing/src/test/java/io/druid/query/topn/TopNBinaryFnBenchmark.java @@ -66,7 +66,7 @@ protected void setUp() throws Exception final List aggregatorFactories = new ArrayList<>(); aggregatorFactories.add(new CountAggregatorFactory("rows")); - aggregatorFactories.add(new LongSumAggregatorFactory("index", "index")); + aggregatorFactories.add(new LongSumAggregatorFactory("index", "index", 1)); for (int i = 1; i < aggCount; i++) { aggregatorFactories.add(new CountAggregatorFactory("rows" + i)); } diff --git a/processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java b/processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java index d5417b1e49d8..85496ed00b0f 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java @@ -44,7 +44,7 @@ public class TopNBinaryFnTest { final CountAggregatorFactory rowsCount = new CountAggregatorFactory("rows"); - final LongSumAggregatorFactory indexLongSum = new LongSumAggregatorFactory("index", "index"); + final LongSumAggregatorFactory indexLongSum = new LongSumAggregatorFactory("index", "index", 1); final ConstantPostAggregator constant = new ConstantPostAggregator("const", 1L); final FieldAccessPostAggregator rowsPostAgg = new FieldAccessPostAggregator("rows", "rows"); final FieldAccessPostAggregator indexPostAgg = new FieldAccessPostAggregator("index", "index"); diff --git a/processing/src/test/java/io/druid/segment/AppendTest.java b/processing/src/test/java/io/druid/segment/AppendTest.java index 2a6afec8b541..c4fff7ade6ca 100644 --- a/processing/src/test/java/io/druid/segment/AppendTest.java +++ b/processing/src/test/java/io/druid/segment/AppendTest.java @@ -66,12 +66,12 @@ public class AppendTest { private static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("index", "index"), + new DoubleSumAggregatorFactory("index", "index", 1), new CountAggregatorFactory("count"), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; private static final AggregatorFactory[] METRIC_AGGS_NO_UNIQ = new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("index", "index"), + new DoubleSumAggregatorFactory("index", "index", 1), new CountAggregatorFactory("count") }; @@ -85,7 +85,7 @@ public class AppendTest final String placementishDimension = "placementish"; final String indexMetric = "index"; final CountAggregatorFactory rowsCount = new CountAggregatorFactory("rows"); - final DoubleSumAggregatorFactory indexDoubleSum = new DoubleSumAggregatorFactory("index", "index"); + final DoubleSumAggregatorFactory indexDoubleSum = new DoubleSumAggregatorFactory("index", "index", 1); final HyperUniquesAggregatorFactory uniques = new HyperUniquesAggregatorFactory("uniques", "quality_uniques"); final ConstantPostAggregator constant = new ConstantPostAggregator("const", 1L); final FieldAccessPostAggregator rowsPostAgg = new FieldAccessPostAggregator("rows", "rows"); diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTest.java b/processing/src/test/java/io/druid/segment/IndexMergerTest.java index a33fa7656bcc..7764d4fd749e 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTest.java @@ -530,9 +530,10 @@ public void testConvertSame() throws Exception new AggregatorFactory[]{ new LongSumAggregatorFactory( "longSum1", - "dim1" + "dim1", + 1 ), - new LongSumAggregatorFactory("longSum2", "dim2") + new LongSumAggregatorFactory("longSum2", "dim2", 1) } ); IncrementalIndexTest.populateIndex(timestamp, toPersist1); @@ -588,9 +589,10 @@ public void testConvertDifferent() throws Exception new AggregatorFactory[]{ new LongSumAggregatorFactory( "longSum1", - "dim1" + "dim1", + 1 ), - new LongSumAggregatorFactory("longSum2", "dim2") + new LongSumAggregatorFactory("longSum2", "dim2", 1) } ); IncrementalIndexTest.populateIndex(timestamp, toPersist1); diff --git a/processing/src/test/java/io/druid/segment/SchemalessIndex.java b/processing/src/test/java/io/druid/segment/SchemalessIndex.java index 588a2764610b..46c593cf35a6 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessIndex.java +++ b/processing/src/test/java/io/druid/segment/SchemalessIndex.java @@ -69,12 +69,12 @@ public class SchemalessIndex private static final String TIMESTAMP = "timestamp"; private static final List METRICS = Arrays.asList("index"); private static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("index", "index"), + new DoubleSumAggregatorFactory("index", "index", 1), new CountAggregatorFactory("count"), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; private static final AggregatorFactory[] METRIC_AGGS_NO_UNIQ = new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("index", "index"), + new DoubleSumAggregatorFactory("index", "index", 1), new CountAggregatorFactory("count") }; diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestFull.java b/processing/src/test/java/io/druid/segment/SchemalessTestFull.java index 4ad2f7419978..93fd22f57346 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestFull.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestFull.java @@ -76,7 +76,7 @@ public class SchemalessTestFull final String placementishDimension = "placementish"; final String indexMetric = "index"; final CountAggregatorFactory rowsCount = new CountAggregatorFactory("rows"); - final DoubleSumAggregatorFactory indexDoubleSum = new DoubleSumAggregatorFactory("index", "index"); + final DoubleSumAggregatorFactory indexDoubleSum = new DoubleSumAggregatorFactory("index", "index", 1); final HyperUniquesAggregatorFactory uniques = new HyperUniquesAggregatorFactory("uniques", "quality_uniques"); final ConstantPostAggregator constant = new ConstantPostAggregator("const", 1L); final FieldAccessPostAggregator rowsPostAgg = new FieldAccessPostAggregator("rows", "rows"); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimple.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimple.java index 77fb21fa1d96..0dc24a18c36e 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimple.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimple.java @@ -102,7 +102,7 @@ public static Collection constructorFeeder() throws IOException final String placementishDimension = "placementish"; final String indexMetric = "index"; final CountAggregatorFactory rowsCount = new CountAggregatorFactory("rows"); - final DoubleSumAggregatorFactory indexDoubleSum = new DoubleSumAggregatorFactory("index", "index"); + final DoubleSumAggregatorFactory indexDoubleSum = new DoubleSumAggregatorFactory("index", "index", 1); final HyperUniquesAggregatorFactory uniques = new HyperUniquesAggregatorFactory("uniques", "quality_uniques"); final ConstantPostAggregator constant = new ConstantPostAggregator("const", 1L); final FieldAccessPostAggregator rowsPostAgg = new FieldAccessPostAggregator("rows", "rows"); diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java index 8a74909c6d70..5d707567162c 100644 --- a/processing/src/test/java/io/druid/segment/TestIndex.java +++ b/processing/src/test/java/io/druid/segment/TestIndex.java @@ -75,7 +75,7 @@ public class TestIndex private static final Logger log = new Logger(TestIndex.class); private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"); private static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ - new DoubleSumAggregatorFactory(METRICS[0], METRICS[0]), + new DoubleSumAggregatorFactory(METRICS[0], METRICS[0], 1), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; private static final IndexSpec indexSpec = new IndexSpec(); diff --git a/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java index b28e493a60f4..31db806445de 100644 --- a/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java @@ -217,13 +217,15 @@ public void testConcurrentAddRead() throws InterruptedException, ExecutionExcept ingestAggregatorFactories.add( new LongSumAggregatorFactory( String.format("sumResult%s", i), - String.format("Dim_%s", i) + String.format("Dim_%s", i), + 1 ) ); ingestAggregatorFactories.add( new DoubleSumAggregatorFactory( String.format("doubleSumResult%s", i), - String.format("Dim_%s", i) + String.format("Dim_%s", i), + 1 ) ); } @@ -234,13 +236,15 @@ public void testConcurrentAddRead() throws InterruptedException, ExecutionExcept queryAggregatorFactories.add( new LongSumAggregatorFactory( String.format("sumResult%s", i), - String.format("sumResult%s", i) + String.format("sumResult%s", i), + 1 ) ); queryAggregatorFactories.add( new DoubleSumAggregatorFactory( String.format("doubleSumResult%s", i), - String.format("doubleSumResult%s", i) + String.format("doubleSumResult%s", i), + 1 ) ); } diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java index 566c047738f5..b938e09aa9c9 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java @@ -74,7 +74,7 @@ public class SpatialFilterBonusTest private static Interval DATA_INTERVAL = new Interval("2013-01-01/2013-01-07"); private static AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("val", "val") + new LongSumAggregatorFactory("val", "val", 1) }; private static List DIMS = Lists.newArrayList("dim", "dim.geo"); private static final IndexMerger INDEX_MERGER = TestHelper.getTestIndexMerger(); @@ -455,7 +455,7 @@ public void testSpatialQuery() .aggregators( Arrays.asList( new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("val", "val") + new LongSumAggregatorFactory("val", "val", 1) ) ) .build(); @@ -507,7 +507,7 @@ public void testSpatialQueryMorePoints() .aggregators( Arrays.asList( new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("val", "val") + new LongSumAggregatorFactory("val", "val", 1) ) ) .build(); diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java index 7d916995ccf2..55eb1cce11f4 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java @@ -78,7 +78,7 @@ public class SpatialFilterTest private static AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("val", "val") + new LongSumAggregatorFactory("val", "val", 1) }; private static List DIMS = Lists.newArrayList("dim", "lat", "long", "lat2", "long2"); @@ -527,7 +527,7 @@ public void testSpatialQuery() .aggregators( Arrays.asList( new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("val", "val") + new LongSumAggregatorFactory("val", "val", 1) ) ) .build(); @@ -580,7 +580,7 @@ public void testSpatialQueryWithOtherSpatialDim() .aggregators( Arrays.asList( new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("val", "val") + new LongSumAggregatorFactory("val", "val", 1) ) ) .build(); @@ -632,7 +632,7 @@ public void testSpatialQueryMorePoints() .aggregators( Arrays.asList( new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("val", "val") + new LongSumAggregatorFactory("val", "val", 1) ) ) .build(); diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index 181b4d2a24cd..9ec04d91eb6f 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -125,7 +125,7 @@ public void testSanity() throws Exception .setInterval(new Interval(0, new DateTime().getMillis())) .addDimension("billy") .addDimension("sally") - .addAggregator(new LongSumAggregatorFactory("cnt", "cnt")) + .addAggregator(new LongSumAggregatorFactory("cnt", "cnt", 1)) .build(), new IncrementalIndexStorageAdapter(index) ); @@ -173,7 +173,7 @@ public void testObjectColumnSelectorOnVaryingColumnSchema() throws Exception .addDimension("billy") .addDimension("sally") .addAggregator( - new LongSumAggregatorFactory("cnt", "cnt") + new LongSumAggregatorFactory("cnt", "cnt", 1) ) .addAggregator( new JavaScriptAggregatorFactory( @@ -314,7 +314,8 @@ public ByteBuffer get() Lists.newArrayList( new LongSumAggregatorFactory( "cnt", - "cnt" + "cnt", + 1 ) ) ) @@ -356,7 +357,7 @@ public void testFilterByNull() throws Exception .setInterval(new Interval(0, new DateTime().getMillis())) .addDimension("billy") .addDimension("sally") - .addAggregator(new LongSumAggregatorFactory("cnt", "cnt")) + .addAggregator(new LongSumAggregatorFactory("cnt", "cnt", 1)) .setDimFilter(DimFilters.dimEquals("sally", (String) null)) .build(), new IncrementalIndexStorageAdapter(index) diff --git a/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java b/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java index 1a67ec12e5b8..514777d7ab59 100644 --- a/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java +++ b/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java @@ -88,13 +88,15 @@ public class OnheapIncrementalIndexBenchmark extends AbstractBenchmark ingestAggregatorFactories.add( new LongSumAggregatorFactory( String.format("sumResult%s", i), - String.format("Dim_%s", i) + String.format("Dim_%s", i), + 1 ) ); ingestAggregatorFactories.add( new DoubleSumAggregatorFactory( String.format("doubleSumResult%s", i), - String.format("Dim_%s", i) + String.format("Dim_%s", i), + 1 ) ); } @@ -246,13 +248,15 @@ public void testConcurrentAddRead() queryAggregatorFactories.add( new LongSumAggregatorFactory( String.format("sumResult%s", i), - String.format("sumResult%s", i) + String.format("sumResult%s", i), + 1 ) ); queryAggregatorFactories.add( new DoubleSumAggregatorFactory( String.format("doubleSumResult%s", i), - String.format("doubleSumResult%s", i) + String.format("doubleSumResult%s", i), + 1 ) ); } diff --git a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java index 3606e055d251..aef2d1d79d66 100644 --- a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java @@ -168,8 +168,8 @@ public class CachingClusteredClientTest private static final int RANDOMNESS = 10; private static final List AGGS = Arrays.asList( new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("imps", "imps"), - new LongSumAggregatorFactory("impers", "imps") + new LongSumAggregatorFactory("imps", "imps", 1), + new LongSumAggregatorFactory("impers", "imps", 1) ); private static final List POST_AGGS = Arrays.asList( new ArithmeticPostAggregator( @@ -199,8 +199,8 @@ public class CachingClusteredClientTest ); private static final List RENAMED_AGGS = Arrays.asList( new CountAggregatorFactory("rows2"), - new LongSumAggregatorFactory("imps", "imps"), - new LongSumAggregatorFactory("impers2", "imps") + new LongSumAggregatorFactory("imps", "imps", 1), + new LongSumAggregatorFactory("impers2", "imps", 1) ); private static final DimFilter DIM_FILTER = null; private static final List RENAMED_POST_AGGS = ImmutableList.of(); diff --git a/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java b/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java index e5cef961e12a..b794a506536f 100644 --- a/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java +++ b/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java @@ -67,8 +67,8 @@ public class CachingQueryRunnerTest private static final List AGGS = Arrays.asList( new CountAggregatorFactory("rows"), - new LongSumAggregatorFactory("imps", "imps"), - new LongSumAggregatorFactory("impers", "imps") + new LongSumAggregatorFactory("imps", "imps", 1), + new LongSumAggregatorFactory("impers", "imps", 1) ); private static final Object[] objects = new Object[]{ diff --git a/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java index 7fc9483f011e..0349a4485c35 100644 --- a/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java @@ -64,8 +64,8 @@ public void testDefaultExclusions() throws Exception "test", parser, new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), + new DoubleSumAggregatorFactory("metric1", "col1", 1), + new DoubleSumAggregatorFactory("metric2", "col2", 1), }, new ArbitraryGranularitySpec(QueryGranularity.DAY, ImmutableList.of(Interval.parse("2014/2015"))), jsonMapper @@ -93,8 +93,8 @@ public void testExplicitInclude() throws Exception "test", parser, new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), + new DoubleSumAggregatorFactory("metric1", "col1", 1), + new DoubleSumAggregatorFactory("metric2", "col2", 1), }, new ArbitraryGranularitySpec(QueryGranularity.DAY, ImmutableList.of(Interval.parse("2014/2015"))), jsonMapper @@ -122,8 +122,8 @@ public void testOverlapMetricNameAndDim() throws Exception "test", parser, new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), + new DoubleSumAggregatorFactory("metric1", "col1", 1), + new DoubleSumAggregatorFactory("metric2", "col2", 1), }, new ArbitraryGranularitySpec(QueryGranularity.DAY, ImmutableList.of(Interval.parse("2014/2015"))), jsonMapper @@ -199,7 +199,7 @@ public void testSerde() throws Exception ), new TypeReference>() {} ), new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1") + new DoubleSumAggregatorFactory("metric1", "col1", 1) }, new ArbitraryGranularitySpec(QueryGranularity.DAY, ImmutableList.of(Interval.parse("2014/2015"))), jsonMapper diff --git a/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java b/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java index c7ef6a99743f..f9e0b2eabd42 100644 --- a/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java +++ b/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java @@ -115,7 +115,7 @@ private void createTestIndex(File segmentDir) throws Exception ); AggregatorFactory[] aggregators = new AggregatorFactory[]{ - new LongSumAggregatorFactory("visited_sum", "visited") + new LongSumAggregatorFactory("visited_sum", "visited", 1) }; IncrementalIndex index = null; diff --git a/server/src/test/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategyTest.java b/server/src/test/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategyTest.java index 5ff245dde7d5..a637d33c4578 100644 --- a/server/src/test/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategyTest.java +++ b/server/src/test/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategyTest.java @@ -109,8 +109,8 @@ public LinkedHashMap getTierToBrokerMap() queryBuilder.aggregators( ImmutableList.of( new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("longSum", "a"), - new DoubleSumAggregatorFactory("doubleSum", "b") + new LongSumAggregatorFactory("longSum", "a", 1), + new DoubleSumAggregatorFactory("doubleSum", "b", 1) ) ).build() ) @@ -125,8 +125,8 @@ public LinkedHashMap getTierToBrokerMap() queryBuilder.aggregators( ImmutableList.of( new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("longSum", "a"), - new DoubleSumAggregatorFactory("doubleSum", "b") + new LongSumAggregatorFactory("longSum", "a", 1), + new DoubleSumAggregatorFactory("doubleSum", "b", 1) ) ).build() ) From 526ed2b3015bb0ebd5b0385af501517565b1986c Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 15 Nov 2015 00:26:52 -0600 Subject: [PATCH 3/4] adding support for square root to arithmetic post aggregator --- .../post/ArithmeticPostAggregator.java | 21 ++++++++++++++++--- .../post/ArithmeticPostAggregatorTest.java | 12 ++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java index cd89f1fdbd62..be68cb16bf2a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -69,7 +69,7 @@ public ArithmeticPostAggregator( ) { Preconditions.checkArgument(fnName != null, "fn cannot not be null"); - Preconditions.checkArgument(fields != null && fields.size() > 1, "Illegal number of fields[%s], must be > 1"); + Preconditions.checkArgument(fields != null && fields.size() >= 1, "Illegal number of fields[%s], must be >= 1"); this.name = name; this.fnName = fnName; @@ -106,7 +106,7 @@ public Object compute(Map values) Iterator fieldsIter = fields.iterator(); double retVal = 0.0; if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); + retVal = op.compute(((Number) fieldsIter.next().compute(values)).doubleValue()); while (fieldsIter.hasNext()) { retVal = op.compute(retVal, ((Number) fieldsIter.next().compute(values)).doubleValue()); } @@ -186,6 +186,13 @@ public double compute(double lhs, double rhs) { return lhs / rhs; } + }, + SQRT("sqrt") + { + public double compute(double val) + { + return Math.sqrt(val); + } }; private static final Map lookupMap = Maps.newHashMap(); @@ -208,7 +215,15 @@ public String getFn() return fn; } - public abstract double compute(double lhs, double rhs); + public double compute(double lhs, double rhs) + { + throw new UnsupportedOperationException("operation not supported"); + } + + public double compute(double val) + { + return val; + } static Ops lookup(String fn) { diff --git a/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java index a005737d3527..bc294551fcb7 100644 --- a/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java @@ -65,7 +65,17 @@ public void testCompute() Assert.assertEquals(18.0, arithmeticPostAggregator.compute(metricValues)); arithmeticPostAggregator = new ArithmeticPostAggregator("divide", "/", postAggregatorList); - Assert.assertEquals(2.0, arithmeticPostAggregator.compute(metricValues)); + Assert.assertEquals(2.0, (double)arithmeticPostAggregator.compute(metricValues), 0.001); + + postAggregatorList = + Lists.newArrayList( + new ConstantPostAggregator( + "roku", 25 + ) + ); + arithmeticPostAggregator = new ArithmeticPostAggregator("sqrt", "sqrt", postAggregatorList); + Assert.assertEquals(5.0, arithmeticPostAggregator.compute(metricValues)); + } @Test From 2ef5582afd5a2c8bcc7f010c3a8153b608e96e37 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 18 Nov 2015 10:55:48 -0600 Subject: [PATCH 4/4] support "function" attribute instead of "exponent" --- .../aggregation/LongSumAggregatorFactory.java | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index a0998c157005..ef31ec0ccc5f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.math.LongMath; import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.segment.ColumnSelectorFactory; @@ -37,13 +38,13 @@ public class LongSumAggregatorFactory implements AggregatorFactory private final String fieldName; private final String name; - private final int exponent; + private final LongFn function; @JsonCreator public LongSumAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldName") final String fieldName, - @JsonProperty("exponent") final Integer exponent + @JsonProperty("function") final LongFn function ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); @@ -51,8 +52,7 @@ public LongSumAggregatorFactory( this.name = name; this.fieldName = fieldName; - this.exponent = exponent == null ? 1 : exponent.intValue(); - Preconditions.checkArgument(this.exponent >= 1, "exponent must be greater or equal to 1"); + this.function = function; } @Override @@ -61,14 +61,14 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) return new LongSumAggregator( name, metricFactory.makeLongColumnSelector(fieldName), - exponent + function ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongSumBufferAggregator(metricFactory.makeLongColumnSelector(fieldName), exponent); + return new LongSumBufferAggregator(metricFactory.makeLongColumnSelector(fieldName), function); } @Override @@ -86,13 +86,13 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongSumAggregatorFactory(name, name, 1); + return new LongSumAggregatorFactory(name, name, null); } @Override public List getRequiredColumns() { - return Arrays.asList(new LongSumAggregatorFactory(fieldName, fieldName, exponent)); + return Arrays.asList(new LongSumAggregatorFactory(fieldName, fieldName, function)); } @Override @@ -191,3 +191,31 @@ public int hashCode() return result; } } + +interface LongFn { + long apply(long x); + byte[] getCacheKey(); +} + +class ExponentLongFn implements LongFn { + + private final int k; + + @JsonCreator + public ExponentLongFn(final Integer k) + { + this.k = k == null? 1 : k.intValue(); + } + + @Override + public long apply(long x) + { + return LongMath.pow(x,k); + } + + @Override + public byte[] getCacheKey() + { + return new byte[0]; + } +}