Skip to content
This repository was archived by the owner on Aug 20, 2025. It is now read-only.

METRON-1382 Run Stellar in a Zeppelin Notebook #884

Closed
nickwallen wants to merge 25 commits into from
Closed

METRON-1382 Run Stellar in a Zeppelin Notebook #884

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
49fba6a
METRON-1382 Run Stellar in a Zeppelin Notebook
nickwallen Dec 18, 2017
162f1f4
Rename StellarShellResult to StellarResult for simplicity
nickwallen Dec 27, 2017
b7cf23e
Added withExpression and withValue to the VariableResult class to sim…
nickwallen Dec 27, 2017
4b65b26
Created a separate interface for notification of events
nickwallen Dec 27, 2017
031f65c
Clean-up
nickwallen Dec 27, 2017
790cdd5
Simple clean-up
nickwallen Dec 27, 2017
18fe04c
Improved the installation instructions
nickwallen Dec 28, 2017
98f73bc
Fixed a bug in the Zeppelin interpreter when no value is returned fro…
nickwallen Dec 28, 2017
fc74720
Should be using System.lineSeparator()
nickwallen Dec 28, 2017
e98b3ef
Added license header to README
nickwallen Dec 31, 2017
2a67e38
Merge remote-tracking branch 'apache/master' into METRON-1382
nickwallen Jan 3, 2018
2b51128
Updated to version 0.4.3
nickwallen Jan 3, 2018
be412d8
Updated the README to clarify usage
nickwallen Jan 3, 2018
2e001af
Fixed the 'only shows first item in list' bug identified by @merrimanr.
nickwallen Jan 4, 2018
9cc1de2
Added a missing license header
nickwallen Jan 4, 2018
ceb55e4
Minor cleanup
nickwallen Jan 4, 2018
30654f3
Updated the interpreter settings to match what I have in the installa…
nickwallen Jan 4, 2018
04d580e
Added more tests because what's better than more tests?
nickwallen Jan 4, 2018
7eb2697
Added <kbd></kbd> embed to make autocomplete instructions more clear.
nickwallen Jan 4, 2018
4c1cd2e
Added tests for isValueNull() behavior
nickwallen Jan 6, 2018
53c1c2d
Added implementation notes as requested by @ottobackwards
nickwallen Jan 6, 2018
3247851
Added note about specific Zeppelin version support in REAEDME
nickwallen Jan 9, 2018
915fdc1
Updated README instructions for adding Stellar interpreter
nickwallen Jan 9, 2018
f779bde
Changes to satisfy checkstyle and findbugs
nickwallen Jan 10, 2018
f6cfe22
Error message then stack trace
nickwallen Jan 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions dependencies_with_url.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,3 +318,19 @@ org.eclipse.persistence:org.eclipse.persistence.asm:jar:2.6.4:compile,EPL 1.0,ht
org.eclipse.persistence:org.eclipse.persistence.core:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink
org.eclipse.persistence:org.eclipse.persistence.jpa.jpql:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink
org.eclipse.persistence:org.eclipse.persistence.jpa:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink

com.google.code.gson:gson:jar:2.2:compile
org.codehaus.plexus:plexus-classworlds:jar:2.4:compile
org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile
org.codehaus.plexus:plexus-interpolation:jar:1.14:compile
org.codehaus.plexus:plexus-utils:jar:2.0.7:compile
org.jsoup:jsoup:jar:1.6.1:compile
org.sonatype.aether:aether-api:jar:1.12:compile
org.sonatype.aether:aether-connector-file:jar:1.12:compile
org.sonatype.aether:aether-connector-wagon:jar:1.12:compile
org.sonatype.aether:aether-impl:jar:1.12:compile
org.sonatype.aether:aether-spi:jar:1.12:compile
org.sonatype.aether:aether-util:jar:1.12:compile
org.sonatype.sisu:sisu-guice:jar:no_aop:3.0.2:compile
org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
2 changes: 1 addition & 1 deletion metron-platform/metron-common/src/main/scripts/stellar
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,4 @@ export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.StellarShell "$@"
java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
57 changes: 31 additions & 26 deletions ...platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,20 @@
*/
package org.apache.metron.management;

import static org.apache.metron.stellar.common.shell.StellarExecutor.CONSOLE;

import com.jakewharton.fliptables.FlipTable;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.text.WordUtils;
import org.apache.metron.stellar.common.shell.VariableResult;
import org.apache.metron.stellar.common.shell.cli.PausableInput;
import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.stellar.dsl.BaseStellarFunction;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.ParseException;
import org.apache.metron.stellar.dsl.Stellar;
import org.apache.metron.stellar.dsl.StellarFunction;
import org.jboss.aesh.console.Console;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
Expand All @@ -31,22 +42,16 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.text.WordUtils;
import org.apache.metron.stellar.common.shell.PausableInput;
import org.apache.metron.stellar.common.shell.StellarExecutor;
import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.stellar.dsl.BaseStellarFunction;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.ParseException;
import org.apache.metron.stellar.dsl.Stellar;
import org.apache.metron.stellar.dsl.StellarFunction;
import org.jboss.aesh.console.Console;
import org.slf4j.LoggerFactory;

import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE;

public class ShellFunctions {
private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

private static Map<String, VariableResult> getVariables(Context context) {
return (Map<String, VariableResult>) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get();
}

@Stellar(
namespace = "SHELL"
,name = "MAP2TABLE"
Expand Down Expand Up @@ -90,19 +95,19 @@ public static class ListVars implements StellarFunction {
@Override
public Object apply(List<Object> args, Context context) throws ParseException {

Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get();
Map<String, VariableResult> variables = getVariables(context);
String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"};
String[][] data = new String[variables.size()][3];
int wordWrap = -1;
if(args.size() > 0) {
wordWrap = ConversionUtils.convert(args.get(0), Integer.class);
}
int i = 0;
for(Map.Entry<String, StellarExecutor.VariableResult> kv : variables.entrySet()) {
StellarExecutor.VariableResult result = kv.getValue();
for(Map.Entry<String, VariableResult> kv : variables.entrySet()) {
VariableResult result = kv.getValue();
data[i++] = new String[] { toWrappedString(kv.getKey().toString(), wordWrap)
, toWrappedString(result.getResult(), wordWrap)
, toWrappedString(result.getExpression(), wordWrap)
, toWrappedString(result.getExpression().get(), wordWrap)
Copy link
Contributor Author

@nickwallen nickwallen Jan 3, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The VariableResult.expression field is now optional. We do not always know the expression that resulted in a value. The ShellFunctions just needed updated to treat this as an Optional.

};
}
return FlipTable.of(headers, data);
Expand Down Expand Up @@ -139,16 +144,16 @@ public static class Var2Map implements StellarFunction {

@Override
public Object apply(List<Object> args, Context context) throws ParseException {
Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get();
Map<String, VariableResult> variables = getVariables(context);
LinkedHashMap<String, String> ret = new LinkedHashMap<>();
for(Object arg : args) {
if(arg == null) {
continue;
}
String variable = (String)arg;
StellarExecutor.VariableResult result = variables.get(variable);
if(result != null && result.getExpression() != null) {
ret.put(variable, result.getExpression());
VariableResult result = variables.get(variable);
if(result != null && result.getExpression().isPresent()) {
ret.put(variable, result.getExpression().orElseGet(() -> ""));
}
}
return ret;
Expand Down Expand Up @@ -177,17 +182,17 @@ public static class GetExpression implements StellarFunction {

@Override
public Object apply(List<Object> args, Context context) throws ParseException {
Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get();
Map<String, VariableResult> variables = getVariables(context);
if(args.size() == 0) {
return null;
}
String variable = (String) args.get(0);
if(variable == null) {
return null;
}
StellarExecutor.VariableResult result = variables.get(variable);
if(result != null && result.getExpression() != null) {
return result.getExpression();
VariableResult result = variables.get(variable);
if(result != null && result.getExpression().isPresent()) {
return result.getExpression().get();
}
return null;
}
Expand Down
1 change: 0 additions & 1 deletion ...ron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.json.simple.parser.JSONParser;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

Expand Down
24 changes: 11 additions & 13 deletions ...-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.stellar.common.StellarProcessor;
import org.apache.metron.stellar.common.shell.StellarExecutor;
import org.apache.metron.stellar.common.shell.VariableResult;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.DefaultVariableResolver;
import org.apache.metron.stellar.dsl.StellarFunctions;
Expand All @@ -44,7 +44,7 @@
public class EnrichmentConfigFunctionsTest {

String configStr = emptyTransformationsConfig();
Map<String, StellarExecutor.VariableResult> variables;
Map<String, VariableResult> variables;
Context context = null;
String enrichmentType = null;
String group = null;
Expand Down Expand Up @@ -73,17 +73,15 @@ public static Collection<Object[]> types() {
});
}



@Before
public void setup() {
variables = ImmutableMap.of(
"upper", new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"),
"lower", new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo")
"upper", VariableResult.withExpression("FOO", "TO_UPPER('foo')"),
"lower", VariableResult.withExpression("foo", "TO_LOWER('FOO')")
);

context = new Context.Builder()
.with(StellarExecutor.SHELL_VARIABLES, () -> variables)
.with(Context.Capabilities.SHELL_VARIABLES, () -> variables)
.build();
}

Expand Down Expand Up @@ -151,7 +149,7 @@ public void testAddEmpty() {
);
Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
Assert.assertEquals(1, size(stellarFunctions));
Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper"));
Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper"));
}

@Test
Expand All @@ -173,8 +171,8 @@ public void testAddHasExisting() {
);
Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
Assert.assertEquals(2, size(stellarFunctions));
Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper"));
Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower"));
Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper"));
Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower"));
}

@Test
Expand Down Expand Up @@ -208,7 +206,7 @@ public void testAddDuplicate() {
);
Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
Assert.assertEquals(1, size(stellarFunctions));
Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper"));
Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper"));
}

@Test
Expand All @@ -229,7 +227,7 @@ public void testRemove() {
);
Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
Assert.assertEquals(1, size(stellarFunctions));
Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower"));
Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower"));
}

@Test
Expand Down Expand Up @@ -270,7 +268,7 @@ public void testRemoveMissing() {
);
Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
Assert.assertEquals(1, size(stellarFunctions));
Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower"));
Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower"));
}

/**
Expand Down
10 changes: 5 additions & 5 deletions ...on-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@

import com.google.common.collect.ImmutableMap;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.stellar.common.shell.VariableResult;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.DefaultVariableResolver;
import org.apache.metron.stellar.dsl.StellarFunctions;
import org.apache.metron.stellar.common.StellarProcessor;
import org.apache.metron.stellar.common.shell.StellarExecutor;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
Expand All @@ -36,7 +36,7 @@

public class IndexingConfigFunctionsTest {

Map<String, StellarExecutor.VariableResult> variables;
Map<String, VariableResult> variables;
Context context = null;

private Object run(String rule, Map<String, Object> variables) {
Expand All @@ -47,12 +47,12 @@ private Object run(String rule, Map<String, Object> variables) {
@Before
public void setup() {
variables = ImmutableMap.of(
"upper", new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"),
"lower", new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo")
"upper", VariableResult.withExpression("FOO", "TO_UPPER('foo')"),
"lower", VariableResult.withExpression("foo", "TO_LOWER('FOO')")
);

context = new Context.Builder()
.with(StellarExecutor.SHELL_VARIABLES, () -> variables)
.with(Context.Capabilities.SHELL_VARIABLES, () -> variables)
.build();
}

Expand Down
10 changes: 5 additions & 5 deletions ...tron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.FieldTransformer;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.stellar.common.shell.VariableResult;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.common.shell.StellarExecutor;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Before;
Expand All @@ -40,17 +40,17 @@ public class ParserConfigFunctionsTest {

String emptyTransformationsConfig = slurp(PARSER_CONFIGS_PATH + "/parsers/bro.json");
String existingTransformationsConfig = slurp(PARSER_CONFIGS_PATH + "/parsers/squid.json");
Map<String, StellarExecutor.VariableResult> variables ;
Map<String, VariableResult> variables ;
Context context = null;
@Before
public void setup() {
variables = ImmutableMap.of(
"upper" , new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"),
"lower" , new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo")
"upper" , VariableResult.withExpression("FOO", "TO_UPPER('foo')"),
"lower" , VariableResult.withExpression("foo", "TO_LOWER('FOO'")
);

context = new Context.Builder()
.with(StellarExecutor.SHELL_VARIABLES , () -> variables)
.with(Context.Capabilities.SHELL_VARIABLES, () -> variables)
.build();
}

Expand Down
22 changes: 12 additions & 10 deletions ...form/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,28 @@

import com.google.common.collect.ImmutableMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.stellar.common.shell.VariableResult;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.common.shell.StellarExecutor;
import org.junit.Assert;
import org.junit.Test;

import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;

public class ShellFunctionsTest {

Map<String, StellarExecutor.VariableResult> variables = ImmutableMap.of(
"var1" , new StellarExecutor.VariableResult("TO_UPPER('casey')", "CASEY"),
"var2" , new StellarExecutor.VariableResult(null, "foo"),
"var3" , new StellarExecutor.VariableResult(null, null),
"var4" , new StellarExecutor.VariableResult("blah", null)
Map<String, VariableResult> variables = ImmutableMap.of(
"var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"),
"var2" , VariableResult.withValue("foo"),
"var3" , VariableResult.withValue(null),
"var4" , VariableResult.withExpression(null, "blah")
);

Context context = new Context.Builder()
.with(StellarExecutor.SHELL_VARIABLES , () -> variables)
.with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
.build();
/**
╔══════════╤═══════╤════════════╗
Expand All @@ -53,10 +54,11 @@ public class ShellFunctionsTest {

@Test
public void testListVarsWithVars() {
Map<String, StellarExecutor.VariableResult> variables = ImmutableMap.of("foo", new StellarExecutor.VariableResult("1 + 1", 2.0));
Map<String, VariableResult> variables = ImmutableMap.of(
"foo", VariableResult.withExpression(2.0, "1 + 1"));

Context context = new Context.Builder()
.with(StellarExecutor.SHELL_VARIABLES , () -> variables)
.with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
.build();
Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
Assert.assertEquals(expectedListWithFoo, out);
Expand All @@ -75,7 +77,7 @@ public void testListVarsWithVars() {
@Test
public void testListVarsWithoutVars() {
Context context = new Context.Builder()
.with(StellarExecutor.SHELL_VARIABLES , () -> new HashMap<>())
.with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>())
.build();
Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
Assert.assertEquals(expectedEmptyList, out);
Expand Down
Loading