Skip to content
This repository was archived by the owner on Aug 20, 2025. It is now read-only.

METRON-426: Stellar does not support scientific notation as a literal #257

Closed
cestella wants to merge 3 commits into from
Closed

METRON-426: Stellar does not support scientific notation as a literal #257

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
03420b8
METRON-426: Stellar does not support scientific notation as a literal
cestella Sep 16, 2016
8fcb968
Updating comment.
cestella Sep 16, 2016
8a8e9b5
Updating test to be less ego-driven.
cestella Sep 20, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...tform/metron-common/src/main/antlr4/org/apache/metron/common/stellar/generated/Stellar.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,9 @@ IN : 'in'
NIN : 'not in'
;
EXISTS : 'exists' | 'EXISTS';
EXPONENT : ('e' | 'E') ( PLUS|MINUS )? ('0'..'9')+;
INT_LITERAL : MINUS? '0'..'9'+ ;
DOUBLE_LITERAL : MINUS? '0'..'9'+'.''0'..'9'+ ;
DOUBLE_LITERAL : MINUS? '0'..'9'+'.''0'..'9'+EXPONENT? ;
IDENTIFIER : [a-zA-Z_][a-zA-Z_\.:0-9]* ;
fragment SCHAR: ~['"\\\r\n];
STRING_LITERAL : '"' SCHAR* '"'
Expand Down
13 changes: 7 additions & 6 deletions metron-platform/metron-common/src/main/java/Stellar.tokens
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,13 @@ RPAREN=28
IN=29
NIN=30
EXISTS=31
INT_LITERAL=32
DOUBLE_LITERAL=33
IDENTIFIER=34
STRING_LITERAL=35
COMMENT=36
WS=37
EXPONENT=32
INT_LITERAL=33
DOUBLE_LITERAL=34
IDENTIFIER=35
STRING_LITERAL=36
COMMENT=37
WS=38
','=1
'=='=7
'!='=8
Expand Down
13 changes: 7 additions & 6 deletions metron-platform/metron-common/src/main/java/StellarLexer.tokens
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,13 @@ RPAREN=28
IN=29
NIN=30
EXISTS=31
INT_LITERAL=32
DOUBLE_LITERAL=33
IDENTIFIER=34
STRING_LITERAL=35
COMMENT=36
WS=37
EXPONENT=32
INT_LITERAL=33
DOUBLE_LITERAL=34
IDENTIFIER=35
STRING_LITERAL=36
COMMENT=37
WS=38
','=1
'=='=7
'!='=8
Expand Down
5 changes: 3 additions & 2 deletions ...on-common/src/main/java/org/apache/metron/common/math/stats/OnlineStatisticsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@ public class OnlineStatisticsProvider implements StatisticsProvider, KryoSeriali
* A sensible default for compression to use in the T-Digest.
* As per https://github.com/tdunning/t-digest/blob/master/src/main/java/com/tdunning/math/stats/TDigest.java#L86
* 100 is a sensible default and the number of centroids retained (to construct the sketch)
* is usually a smallish (usually < 10) multiple of the compression.
* is usually a smallish (usually < 10) multiple of the compression. However, we have found through some
* testing that 150 gives a bit finer granularity for smaller numbers of data points.
*/
public static final int COMPRESSION = 100;
public static final int COMPRESSION = 150;
Copy link
Contributor

@justinleet justinleet Sep 16, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you update the comment to reflect the compression default change?

Copy link
Member Author

@cestella cestella Sep 16, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yep



/**
Expand Down
209 changes: 108 additions & 101 deletions .../metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarLexer.java
View file
Open in desktop

Large diffs are not rendered by default.

21 changes: 11 additions & 10 deletions ...metron-common/src/main/java/org/apache/metron/common/stellar/generated/StellarParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,9 @@ public class StellarParser extends Parser {
COMMA=1, AND=2, OR=3, NOT=4, TRUE=5, FALSE=6, EQ=7, NEQ=8, LT=9, LTE=10,
GT=11, GTE=12, QUESTION=13, COLON=14, IF=15, THEN=16, ELSE=17, NULL=18,
MINUS=19, PLUS=20, DIV=21, MUL=22, LBRACE=23, RBRACE=24, LBRACKET=25,
RBRACKET=26, LPAREN=27, RPAREN=28, IN=29, NIN=30, EXISTS=31, INT_LITERAL=32,
DOUBLE_LITERAL=33, IDENTIFIER=34, STRING_LITERAL=35, COMMENT=36, WS=37;
RBRACKET=26, LPAREN=27, RPAREN=28, IN=29, NIN=30, EXISTS=31, EXPONENT=32,
INT_LITERAL=33, DOUBLE_LITERAL=34, IDENTIFIER=35, STRING_LITERAL=36, COMMENT=37,
WS=38;
public static final int
RULE_transformation = 0, RULE_transformation_expr = 1, RULE_conditional_expr = 2,
RULE_comparison_expr = 3, RULE_transformation_entity = 4, RULE_comp_operator = 5,
Expand All @@ -65,8 +66,8 @@ public class StellarParser extends Parser {
null, "COMMA", "AND", "OR", "NOT", "TRUE", "FALSE", "EQ", "NEQ", "LT",
"LTE", "GT", "GTE", "QUESTION", "COLON", "IF", "THEN", "ELSE", "NULL",
"MINUS", "PLUS", "DIV", "MUL", "LBRACE", "RBRACE", "LBRACKET", "RBRACKET",
"LPAREN", "RPAREN", "IN", "NIN", "EXISTS", "INT_LITERAL", "DOUBLE_LITERAL",
"IDENTIFIER", "STRING_LITERAL", "COMMENT", "WS"
"LPAREN", "RPAREN", "IN", "NIN", "EXISTS", "EXPONENT", "INT_LITERAL",
"DOUBLE_LITERAL", "IDENTIFIER", "STRING_LITERAL", "COMMENT", "WS"
};
public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);

Expand Down Expand Up @@ -2091,7 +2092,7 @@ private boolean arithmetic_expr_mul_sempred(Arithmetic_expr_mulContext _localctx
}

public static final String _serializedATN =
"\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3\'\u00e4\4\2\t\2\4"+
"\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3(\u00e4\4\2\t\2\4"+
"\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
"\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
"\4\23\t\23\3\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3\62\n\3\3\4"+
Expand Down Expand Up @@ -2150,16 +2151,16 @@ private boolean arithmetic_expr_mul_sempred(Arithmetic_expr_mulContext _localctx
"\u00b9\7\30\2\2\u00b9\u00be\5\36\20\5\u00ba\u00bb\f\3\2\2\u00bb\u00bc"+
"\7\27\2\2\u00bc\u00be\5\36\20\4\u00bd\u00b7\3\2\2\2\u00bd\u00ba\3\2\2"+
"\2\u00be\u00c1\3\2\2\2\u00bf\u00bd\3\2\2\2\u00bf\u00c0\3\2\2\2\u00c0\37"+
"\3\2\2\2\u00c1\u00bf\3\2\2\2\u00c2\u00c3\7$\2\2\u00c3\u00c4\5\22\n\2\u00c4"+
"!\3\2\2\2\u00c5\u00d2\5 \21\2\u00c6\u00d2\7#\2\2\u00c7\u00d2\7\"\2\2\u00c8"+
"\u00d2\7$\2\2\u00c9\u00ca\7\35\2\2\u00ca\u00cb\5\34\17\2\u00cb\u00cc\7"+
"\3\2\2\2\u00c1\u00bf\3\2\2\2\u00c2\u00c3\7%\2\2\u00c3\u00c4\5\22\n\2\u00c4"+
"!\3\2\2\2\u00c5\u00d2\5 \21\2\u00c6\u00d2\7$\2\2\u00c7\u00d2\7#\2\2\u00c8"+
"\u00d2\7%\2\2\u00c9\u00ca\7\35\2\2\u00ca\u00cb\5\34\17\2\u00cb\u00cc\7"+
"\36\2\2\u00cc\u00d2\3\2\2\2\u00cd\u00ce\7\35\2\2\u00ce\u00cf\5\6\4\2\u00cf"+
"\u00d0\7\36\2\2\u00d0\u00d2\3\2\2\2\u00d1\u00c5\3\2\2\2\u00d1\u00c6\3"+
"\2\2\2\u00d1\u00c7\3\2\2\2\u00d1\u00c8\3\2\2\2\u00d1\u00c9\3\2\2\2\u00d1"+
"\u00cd\3\2\2\2\u00d2#\3\2\2\2\u00d3\u00e2\t\5\2\2\u00d4\u00e2\5\34\17"+
"\2\u00d5\u00e2\7%\2\2\u00d6\u00e2\5\26\f\2\u00d7\u00e2\5\32\16\2\u00d8"+
"\2\u00d5\u00e2\7&\2\2\u00d6\u00e2\5\26\f\2\u00d7\u00e2\5\32\16\2\u00d8"+
"\u00e2\7\24\2\2\u00d9\u00da\7!\2\2\u00da\u00db\7\35\2\2\u00db\u00dc\7"+
"$\2\2\u00dc\u00e2\7\36\2\2\u00dd\u00de\7\35\2\2\u00de\u00df\5\6\4\2\u00df"+
"%\2\2\u00dc\u00e2\7\36\2\2\u00dd\u00de\7\35\2\2\u00de\u00df\5\6\4\2\u00df"+
"\u00e0\7\36\2\2\u00e0\u00e2\3\2\2\2\u00e1\u00d3\3\2\2\2\u00e1\u00d4\3"+
"\2\2\2\u00e1\u00d5\3\2\2\2\u00e1\u00d6\3\2\2\2\u00e1\u00d7\3\2\2\2\u00e1"+
"\u00d8\3\2\2\2\u00e1\u00d9\3\2\2\2\u00e1\u00dd\3\2\2\2\u00e2%\3\2\2\2"+
Expand Down
10 changes: 9 additions & 1 deletion ...common/src/test/java/org/apache/metron/common/stellar/StellarStatisticsFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
package org.apache.metron.common.stellar;

import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import org.apache.commons.math3.random.GaussianRandomGenerator;
import org.apache.commons.math3.random.MersenneTwister;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
Expand Down Expand Up @@ -177,14 +178,21 @@ public void testOverflow() throws Exception {
run(format("STATS_ADD(STATS_INIT(), %f)", (Double.MAX_VALUE + 1)), new HashMap<>());
}

@Test
public void ensureDeterminism() throws Exception {
for(int i = 0;i < 20;++i) {
testMergeProviders();
}
}

@Test
public void testMergeProviders() throws Exception {
List<StatisticsProvider> providers = new ArrayList<>();
/*
Create 10 providers, each with a sample drawn from a gaussian distribution.
Update the reference stats from commons math to ensure we are
*/
GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L));
GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(1L));
SummaryStatistics sStatistics= new SummaryStatistics();
DescriptiveStatistics dStatistics = new DescriptiveStatistics();
for(int i = 0;i < 10;++i) {
Expand Down
16 changes: 16 additions & 0 deletions ...on-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,22 @@ public void testMapConstant() {
}
}

@Test
public void testArithmetic() {
{
String query = "1 + 2";
Assert.assertEquals(3, ((Number)run(query, new HashMap<>())).doubleValue(), 1e-3);
}
{
String query = "1.2 + 2";
Assert.assertEquals(3.2, ((Number)run(query, new HashMap<>())).doubleValue(), 1e-3);
}
{
String query = "1.2e-3 + 2";
Assert.assertEquals(1.2e-3 + 2, ((Number)run(query, new HashMap<>())).doubleValue(), 1e-3);
}
Copy link
Contributor

@nickwallen nickwallen Sep 20, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The ImmutableMap.of("casey","casey")(besides promoting your global brand ;) ) do nothing for these tests, right? It'd be more clear, if we removed those. Everything else looks good. I'm a +1 if we can take care of this small issue.

Copy link
Member Author

@cestella cestella Sep 20, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You got it. :)

}

@Test
public void testIfThenElse() {
{
Expand Down