diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 38a9f5e70e..d1c0cf6a1a 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -318,3 +318,19 @@ org.eclipse.persistence:org.eclipse.persistence.asm:jar:2.6.4:compile,EPL 1.0,ht org.eclipse.persistence:org.eclipse.persistence.core:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink org.eclipse.persistence:org.eclipse.persistence.jpa.jpql:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink org.eclipse.persistence:org.eclipse.persistence.jpa:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink + +com.google.code.gson:gson:jar:2.2:compile + org.codehaus.plexus:plexus-classworlds:jar:2.4:compile + org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile + org.codehaus.plexus:plexus-interpolation:jar:1.14:compile + org.codehaus.plexus:plexus-utils:jar:2.0.7:compile + org.jsoup:jsoup:jar:1.6.1:compile + org.sonatype.aether:aether-api:jar:1.12:compile + org.sonatype.aether:aether-connector-file:jar:1.12:compile + org.sonatype.aether:aether-connector-wagon:jar:1.12:compile + org.sonatype.aether:aether-impl:jar:1.12:compile + org.sonatype.aether:aether-spi:jar:1.12:compile + org.sonatype.aether:aether-util:jar:1.12:compile + org.sonatype.sisu:sisu-guice:jar:no_aop:3.0.2:compile + org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile + org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile \ No newline at end of file diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar index 3f53c49bf6..431ed7be8d 100644 --- a/metron-platform/metron-common/src/main/scripts/stellar +++ b/metron-platform/metron-common/src/main/scripts/stellar @@ -33,4 +33,4 @@ export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar) export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar) -java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.StellarShell "$@" +java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java index 52ff157b73..afac7f0160 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java @@ -17,9 +17,20 @@ */ package org.apache.metron.management; -import static org.apache.metron.stellar.common.shell.StellarExecutor.CONSOLE; - import com.jakewharton.fliptables.FlipTable; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.text.WordUtils; +import org.apache.metron.stellar.common.shell.VariableResult; +import org.apache.metron.stellar.common.shell.cli.PausableInput; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.jboss.aesh.console.Console; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -31,22 +42,16 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.text.WordUtils; -import org.apache.metron.stellar.common.shell.PausableInput; -import org.apache.metron.stellar.common.shell.StellarExecutor; -import org.apache.metron.stellar.common.utils.ConversionUtils; -import org.apache.metron.stellar.dsl.BaseStellarFunction; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.dsl.StellarFunction; -import org.jboss.aesh.console.Console; -import org.slf4j.LoggerFactory; + +import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE; public class ShellFunctions { private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static Map getVariables(Context context) { + return (Map) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get(); + } + @Stellar( namespace = "SHELL" ,name = "MAP2TABLE" @@ -90,7 +95,7 @@ public static class ListVars implements StellarFunction { @Override public Object apply(List args, Context context) throws ParseException { - Map variables = (Map) context.getCapability(StellarExecutor.SHELL_VARIABLES).get(); + Map variables = getVariables(context); String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"}; String[][] data = new String[variables.size()][3]; int wordWrap = -1; @@ -98,11 +103,11 @@ public Object apply(List args, Context context) throws ParseException { wordWrap = ConversionUtils.convert(args.get(0), Integer.class); } int i = 0; - for(Map.Entry kv : variables.entrySet()) { - StellarExecutor.VariableResult result = kv.getValue(); + for(Map.Entry kv : variables.entrySet()) { + VariableResult result = kv.getValue(); data[i++] = new String[] { toWrappedString(kv.getKey().toString(), wordWrap) , toWrappedString(result.getResult(), wordWrap) - , toWrappedString(result.getExpression(), wordWrap) + , toWrappedString(result.getExpression().get(), wordWrap) }; } return FlipTable.of(headers, data); @@ -139,16 +144,16 @@ public static class Var2Map implements StellarFunction { @Override public Object apply(List args, Context context) throws ParseException { - Map variables = (Map) context.getCapability(StellarExecutor.SHELL_VARIABLES).get(); + Map variables = getVariables(context); LinkedHashMap ret = new LinkedHashMap<>(); for(Object arg : args) { if(arg == null) { continue; } String variable = (String)arg; - StellarExecutor.VariableResult result = variables.get(variable); - if(result != null && result.getExpression() != null) { - ret.put(variable, result.getExpression()); + VariableResult result = variables.get(variable); + if(result != null && result.getExpression().isPresent()) { + ret.put(variable, result.getExpression().orElseGet(() -> "")); } } return ret; @@ -177,7 +182,7 @@ public static class GetExpression implements StellarFunction { @Override public Object apply(List args, Context context) throws ParseException { - Map variables = (Map) context.getCapability(StellarExecutor.SHELL_VARIABLES).get(); + Map variables = getVariables(context); if(args.size() == 0) { return null; } @@ -185,9 +190,9 @@ public Object apply(List args, Context context) throws ParseException { if(variable == null) { return null; } - StellarExecutor.VariableResult result = variables.get(variable); - if(result != null && result.getExpression() != null) { - return result.getExpression(); + VariableResult result = variables.get(variable); + if(result != null && result.getExpression().isPresent()) { + return result.getExpression().get(); } return null; } diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java index 31eeafec23..19200317bb 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java @@ -31,7 +31,6 @@ import org.json.simple.parser.JSONParser; import org.json.simple.JSONObject; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java index 51a24b3cc0..d552aa1658 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java @@ -23,7 +23,7 @@ import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.shell.StellarExecutor; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; @@ -44,7 +44,7 @@ public class EnrichmentConfigFunctionsTest { String configStr = emptyTransformationsConfig(); - Map variables; + Map variables; Context context = null; String enrichmentType = null; String group = null; @@ -73,17 +73,15 @@ public static Collection types() { }); } - - @Before public void setup() { variables = ImmutableMap.of( - "upper", new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"), - "lower", new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo") + "upper", VariableResult.withExpression("FOO", "TO_UPPER('foo')"), + "lower", VariableResult.withExpression("foo", "TO_LOWER('FOO')") ); context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES, () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES, () -> variables) .build(); } @@ -151,7 +149,7 @@ public void testAddEmpty() { ); Map stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(1, size(stellarFunctions)); - Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper")); + Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper")); } @Test @@ -173,8 +171,8 @@ public void testAddHasExisting() { ); Map stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(2, size(stellarFunctions)); - Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper")); - Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower")); + Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper")); + Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower")); } @Test @@ -208,7 +206,7 @@ public void testAddDuplicate() { ); Map stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(1, size(stellarFunctions)); - Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper")); + Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper")); } @Test @@ -229,7 +227,7 @@ public void testRemove() { ); Map stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(1, size(stellarFunctions)); - Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower")); + Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower")); } @Test @@ -270,7 +268,7 @@ public void testRemoveMissing() { ); Map stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(1, size(stellarFunctions)); - Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower")); + Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower")); } /** diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java index 29c80b8202..c931df5123 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java @@ -19,11 +19,11 @@ import com.google.common.collect.ImmutableMap; import org.apache.metron.common.configuration.IndexingConfigurations; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.shell.StellarExecutor; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -36,7 +36,7 @@ public class IndexingConfigFunctionsTest { - Map variables; + Map variables; Context context = null; private Object run(String rule, Map variables) { @@ -47,12 +47,12 @@ private Object run(String rule, Map variables) { @Before public void setup() { variables = ImmutableMap.of( - "upper", new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"), - "lower", new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo") + "upper", VariableResult.withExpression("FOO", "TO_UPPER('foo')"), + "lower", VariableResult.withExpression("foo", "TO_LOWER('FOO')") ); context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES, () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES, () -> variables) .build(); } diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java index f830e626b2..e27a7ad446 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java @@ -21,8 +21,8 @@ import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.common.shell.StellarExecutor; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Before; @@ -40,17 +40,17 @@ public class ParserConfigFunctionsTest { String emptyTransformationsConfig = slurp(PARSER_CONFIGS_PATH + "/parsers/bro.json"); String existingTransformationsConfig = slurp(PARSER_CONFIGS_PATH + "/parsers/squid.json"); - Map variables ; + Map variables ; Context context = null; @Before public void setup() { variables = ImmutableMap.of( - "upper" , new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"), - "lower" , new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo") + "upper" , VariableResult.withExpression("FOO", "TO_UPPER('foo')"), + "lower" , VariableResult.withExpression("foo", "TO_LOWER('FOO'") ); context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES , () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES, () -> variables) .build(); } diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java index 401454e763..83c2bce41a 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java @@ -19,27 +19,28 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.common.shell.StellarExecutor; import org.junit.Assert; import org.junit.Test; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; public class ShellFunctionsTest { - Map variables = ImmutableMap.of( - "var1" , new StellarExecutor.VariableResult("TO_UPPER('casey')", "CASEY"), - "var2" , new StellarExecutor.VariableResult(null, "foo"), - "var3" , new StellarExecutor.VariableResult(null, null), - "var4" , new StellarExecutor.VariableResult("blah", null) + Map variables = ImmutableMap.of( + "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"), + "var2" , VariableResult.withValue("foo"), + "var3" , VariableResult.withValue(null), + "var4" , VariableResult.withExpression(null, "blah") ); Context context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES , () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES , () -> variables) .build(); /** ╔══════════╤═══════╤════════════╗ @@ -53,10 +54,11 @@ public class ShellFunctionsTest { @Test public void testListVarsWithVars() { - Map variables = ImmutableMap.of("foo", new StellarExecutor.VariableResult("1 + 1", 2.0)); + Map variables = ImmutableMap.of( + "foo", VariableResult.withExpression(2.0, "1 + 1")); Context context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES , () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES , () -> variables) .build(); Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context); Assert.assertEquals(expectedListWithFoo, out); @@ -75,7 +77,7 @@ public void testListVarsWithVars() { @Test public void testListVarsWithoutVars() { Context context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES , () -> new HashMap<>()) + .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>()) .build(); Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context); Assert.assertEquals(expectedEmptyList, out); diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java index 2b154d8ec2..e281038c96 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java @@ -22,7 +22,7 @@ import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.shell.StellarExecutor; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; import org.apache.metron.stellar.dsl.MapVariableResolver; @@ -44,18 +44,18 @@ public class ThreatTriageFunctionsTest { String configStr = emptyTransformationsConfig(); - Map variables; + Map variables; Context context = null; @Before public void setup() { variables = ImmutableMap.of( - "less", new StellarExecutor.VariableResult("1 < 2", true), - "greater", new StellarExecutor.VariableResult("1 > 2", false) + "less", VariableResult.withExpression(true, "1 < 2"), + "greater", VariableResult.withExpression(false, "1 > 2") ); context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES, () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES, () -> variables) .build(); } @@ -127,7 +127,7 @@ public void testAddEmpty() { List triageRules = getTriageRules(newConfig); Assert.assertEquals(1, triageRules.size()); RiskLevelRule rule = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() ); Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); } @@ -165,11 +165,11 @@ public void testAddHasExisting() { List triageRules = getTriageRules(newConfig); Assert.assertEquals(2, triageRules.size()); RiskLevelRule less = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), less.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), less.getRule() ); Assert.assertEquals(10.0, less.getScore().doubleValue(), 1e-6 ); RiskLevelRule greater = triageRules.get(1); - Assert.assertEquals(variables.get("greater").getExpression(), greater.getRule() ); + Assert.assertEquals(variables.get("greater").getExpression().get(), greater.getRule() ); Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 ); } @@ -200,7 +200,7 @@ public void testAddDuplicate() { List triageRules = getTriageRules(newConfig); Assert.assertEquals(1, triageRules.size()); RiskLevelRule rule = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() ); Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); } @@ -256,7 +256,7 @@ public void testRemove() { List triageRules = getTriageRules(newConfig); Assert.assertEquals(1, triageRules.size()); RiskLevelRule rule = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() ); Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); } @@ -281,7 +281,7 @@ public void testRemoveWithEngine() { List triageRules = engine.getRiskLevelRules(); Assert.assertEquals(1, triageRules.size()); RiskLevelRule rule = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() ); Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); } @@ -320,11 +320,11 @@ public void testRemoveMissing() { List triageRules = getTriageRules(newConfig); Assert.assertEquals(2, triageRules.size()); RiskLevelRule less = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), less.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), less.getRule() ); Assert.assertEquals(10.0, less.getScore().doubleValue(), 1e-6 ); RiskLevelRule greater = triageRules.get(1); - Assert.assertEquals(variables.get("greater").getExpression(), greater.getRule() ); + Assert.assertEquals(variables.get("greater").getExpression().get(), greater.getRule() ); Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 ); } diff --git a/metron-stellar/pom.xml b/metron-stellar/pom.xml index c2181fd451..6b5cb672dd 100644 --- a/metron-stellar/pom.xml +++ b/metron-stellar/pom.xml @@ -43,6 +43,7 @@ stellar-3rd-party-example stellar-common + stellar-zeppelin diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 6c420b6ef8..0da3a7da0c 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -29,6 +29,7 @@ For a variety of components (threat intelligence triage and field transformation * [Variable Assignment](#variable-assignment) * [Magic Commands](#magic-commands) * [Advanced Usage](#advanced-usage) + * [Implementation](#implementation) * [Stellar Configuration](#stellar-configuration) @@ -1377,7 +1378,7 @@ IS_EMAIL To run the Stellar Shell directly from the Metron source code, run a command like the following. Ensure that Metron has already been built and installed with `mvn clean install -DskipTests`. ``` $ mvn exec:java \ - -Dexec.mainClass="org.apache.metron.stellar.common.shell.StellarShell" \ + -Dexec.mainClass="org.apache.metron.stellar.common.shell.cli.StellarShell" \ -pl metron-platform/metron-enrichment ... Stellar, Go! @@ -1393,7 +1394,7 @@ This can be useful for troubleshooting function resolution problems. The previo ``` $ mvn exec:java \ - -Dexec.mainClass="org.apache.metron.stellar.common.shell.StellarShell" \ + -Dexec.mainClass="org.apache.metron.stellar.common.shell.cli.StellarShell" \ -pl metron-analytics/metron-profiler ... Stellar, Go! @@ -1403,6 +1404,36 @@ Please note that functions are loading lazily in the background and will be unav ABS, APPEND_IF_MISSING, BIN, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, CHOMP, CHOP, COUNT_MATCHES, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, FILL_LEFT, FILL_RIGHT, FILTER, FORMAT, GET, GET_FIRST, GET_LAST, HLLP_ADD, HLLP_CARDINALITY, HLLP_INIT, HLLP_MERGE, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, JOIN, LENGTH, LIST_ADD, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP, MAP_EXISTS, MAP_GET, MONTH, OUTLIER_MAD_ADD, OUTLIER_MAD_SCORE, OUTLIER_MAD_STATE_MERGE, PREPEND_IF_MISSING, PROFILE_FIXED, PROFILE_GET, PROFILE_WINDOW, PROTOCOL_TO_NAME, REDUCE, REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_BIN, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS_SUM, STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, STRING_ENTROPY, SYSTEM_ENV_GET, SYSTEM_PROPERTY_GET, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, TO_INTEGER, TO_LONG, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR ``` +### Implementation + +The Stellar Shell can be executed both from the command line and from within a Stellar Notebook. The behavior and underlying implementation of the behavior is exactly the same across these two environments. + +#### `org.apache.metron.stellar.common.shell` + +This package contains classes that are reused across both the CLI and Zeppelin shell environments. + +* `StellarShellExecutor` Executes Stellar in a shell-like environment. Provides the Stellar language extensions like variable assignment, comments, magics, and doc strings that are only accessible in the shell. + +* `StellarAutoCompleter` Handles auto-completion for Stellar. + +* `StellarExecutorListeners` An event listener that can be notified when variables, functions, and specials are defined. This is how a `StellarAutoCompleter` is notified throughout the life of a shell session. + +#### `org.apache.metron.stellar.common.shell.specials` + +All Stellar language extensions are contained within this package. + +* `SpecialCommand` The interface for all special commands. A 'special command' is anything that is not directly provided by the Stellar language itself. That includes variable assignment, comments, doc strings, magics, and quit. + +#### `org.apache.metron.stellar.common.shell.cli` + +This package contains classes that are specific to the CLI-driven REPL. + +* `StellarShell` This is the main class that drives the CLI REPL. All functionality not related to the command line interface is performed by the shared logic in `org.apache.metron.stellar.common.shell`. + +#### `org.apache.metron.stellar.zeppelin` + +This package is contained within the `stellar-zeppelin` project and performs all logic for interfacing with Zeppelin. Again, all functionality not related to Zeppelin is performed by the shared logic in `org.apache.metron.stellar.common.shell`. + ## Stellar Configuration Stellar can be configured in a variety of ways from the [Global Configuration](../../metron-platform/metron-common/README.md#global-configuration). diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml index fbb1a05d56..a5dd20b01d 100644 --- a/metron-stellar/stellar-common/pom.xml +++ b/metron-stellar/stellar-common/pom.xml @@ -29,7 +29,6 @@ 1.10 - org.apache.hadoop hadoop-auth @@ -240,7 +239,6 @@ - org.apache.maven.plugins maven-shade-plugin diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java index e77211af04..587b5585ae 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java @@ -38,7 +38,9 @@ public String getStatement() { } public static boolean isAssignment(String statement) { - return statement != null && statement.contains(":="); + return statement != null && + statement.contains(":=") && // has the assignment operator + !statement.trim().startsWith("%"); // not a magic like %define x := 2 } public static StellarAssignment from(String statement) { diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java new file mode 100644 index 0000000000..cf52955d21 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java @@ -0,0 +1,220 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import org.apache.commons.collections4.IterableUtils; +import org.apache.commons.collections4.trie.PatriciaTrie; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + + +/** + * Provides auto-completion for Stellar. + */ +public class DefaultStellarAutoCompleter implements StellarAutoCompleter { + + enum OperationType { + DOC, + MAGIC, + NORMAL + } + + enum AutoCompleteType implements AutoCompleteTransformation { + FUNCTION((type, key) -> { + if(OperationType.DOC == type) { + return "?" + key; + + } else if(OperationType.NORMAL == type) { + return key + "("; + } + + return key; + }), + VARIABLE((type, key) -> key), + TOKEN((type, key) -> key); + + AutoCompleteTransformation transform; + + AutoCompleteType(AutoCompleteTransformation transform) { + this.transform = transform; + } + + @Override + public String transform(OperationType type, String key) { + return transform.transform(type, key); + } + } + + /** + * Prefix tree index of auto-completes. + */ + private PatriciaTrie autocompleteIndex; + + private ReadWriteLock indexLock = new ReentrantReadWriteLock(); + + public interface AutoCompleteTransformation { + String transform(OperationType type, String key); + } + + public DefaultStellarAutoCompleter() { + this.autocompleteIndex = initializeIndex(); + } + + @Override + public Iterable autoComplete(String buffer) { + Iterable candidates = IterableUtils.emptyIterable(); + + final String lastToken = getLastToken(buffer); + if(StringUtils.isNotEmpty(lastToken)) { + + if (isDoc(lastToken)) { + candidates = autoCompleteDoc(lastToken.substring(1)); + + } else if (isMagic(lastToken)) { + candidates = autoCompleteMagic(lastToken); + + } else { + candidates = autoCompleteNormal(lastToken); + } + } + + return candidates; + } + + /** + * Is a given expression a built-in magic? + * @param expression The expression. + */ + private boolean isMagic(String expression) { + return StringUtils.startsWith(expression, "%"); + } + + /** + * Is a given expression asking for function documentation? + * @param expression The expression. + */ + private boolean isDoc(String expression) { + return StringUtils.startsWith(expression, "?"); + } + + /** + * Auto-completes a partial Stellar expression + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable autoCompleteNormal(String buffer) { + return autoComplete(buffer, OperationType.NORMAL); + } + + /** + * Auto-completes a partial doc command. + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable autoCompleteDoc(String buffer) { + return autoComplete(buffer, OperationType.DOC); + } + + /** + * Auto-completes a partial magic commands. + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable autoCompleteMagic(String buffer) { + return autoComplete(buffer, OperationType.MAGIC); + } + + /** + * Returns a list of viable candidates for auto-completion. + * @param buffer The current buffer. + * @param opType The type of operation needing auto-completion. + * @return Viable candidates for auto-completion. + */ + private Iterable autoComplete(String buffer, final OperationType opType) { + indexLock.readLock().lock(); + try { + SortedMap ret = autocompleteIndex.prefixMap(buffer); + if (ret.isEmpty()) { + return new ArrayList<>(); + } + return Iterables.transform(ret.entrySet(), kv -> kv.getValue().transform(opType, kv.getKey())); + } + finally { + indexLock.readLock().unlock(); + } + } + + /** + * Adds a candidate for auto-completing function names. + * @param name The name of the function candidate. + */ + @Override + public void addCandidateFunction(String name) { + add(name, AutoCompleteType.FUNCTION); + } + + /** + * Adds a candidate for auto-completing variable names. + * @param name The name of the function candidate. + */ + @Override + public void addCandidateVariable(String name) { + add(name, AutoCompleteType.VARIABLE); + } + + /** + * Add a candidate for auto-completion. + * @param name The name of the candidate. + * @param type The type of candidate. + */ + private void add(String name, AutoCompleteType type) { + if(StringUtils.isNotBlank(name)) { + // add the candidate to the auto-complete index + indexLock.writeLock().lock(); + try { + this.autocompleteIndex.put(name, type); + } finally { + indexLock.writeLock().unlock(); + } + } + } + + private PatriciaTrie initializeIndex() { + Map index = new HashMap<>(); + index.put("==", AutoCompleteType.TOKEN); + index.put(">=", AutoCompleteType.TOKEN); + index.put("<=", AutoCompleteType.TOKEN); + + return new PatriciaTrie<>(index); + } + + private static String getLastToken(String buffer) { + String lastToken = Iterables.getLast(Splitter.on(" ").split(buffer), null); + return lastToken.trim(); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java new file mode 100644 index 0000000000..f83bb9eb92 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java @@ -0,0 +1,413 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Maps; +import org.apache.commons.collections.map.UnmodifiableMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.configuration.ConfigurationsUtils; +import org.apache.metron.stellar.common.shell.StellarExecutionListeners.FunctionDefinedListener; +import org.apache.metron.stellar.common.shell.StellarExecutionListeners.SpecialDefinedListener; +import org.apache.metron.stellar.common.shell.StellarExecutionListeners.VariableDefinedListener; +import org.apache.metron.stellar.common.shell.specials.AssignmentCommand; +import org.apache.metron.stellar.common.shell.specials.Comment; +import org.apache.metron.stellar.common.shell.specials.DocCommand; +import org.apache.metron.stellar.common.shell.specials.MagicDefineGlobal; +import org.apache.metron.stellar.common.shell.specials.MagicListFunctions; +import org.apache.metron.stellar.common.shell.specials.MagicListGlobals; +import org.apache.metron.stellar.common.shell.specials.MagicListVariables; +import org.apache.metron.stellar.common.shell.specials.MagicUndefineGlobal; +import org.apache.metron.stellar.common.shell.specials.QuitCommand; +import org.apache.metron.stellar.common.shell.specials.SpecialCommand; +import org.apache.metron.stellar.common.utils.JSONUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.StellarFunctionInfo; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static org.apache.metron.stellar.common.configuration.ConfigurationsUtils.readGlobalConfigBytesFromZookeeper; +import static org.apache.metron.stellar.common.shell.StellarResult.noop; +import static org.apache.metron.stellar.common.shell.StellarResult.error; +import static org.apache.metron.stellar.common.shell.StellarResult.success; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.STELLAR_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.ZOOKEEPER_CLIENT; + +/** + * Default implementation of a StellarShellExecutor. + */ +public class DefaultStellarShellExecutor implements StellarShellExecutor { + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String SHELL_VARIABLES = "shellVariables"; + + /** + * The variables known by Stellar. + */ + private Map variables; + + /** + * The function resolver. + */ + private FunctionResolver functionResolver; + + /** + * A Zookeeper client. Only defined if given a valid Zookeeper URL. + */ + private Optional zkClient; + + /** + * A registry of all special commands; like %magic, ?doc, and quit. + */ + private List specials; + + /** + * The Stellar execution context. + */ + private Context context; + + /** + * Listeners that are notified when a function is defined. + */ + private List functionListeners; + + /** + * Listeners that are notified when a variable is defined. + */ + private List variableListeners; + + /** + * Listeners that are notified when a special command is defined. + */ + private List specialListeners; + + + public DefaultStellarShellExecutor( + FunctionResolver functionResolver, + Properties properties, + Optional zookeeperUrl, + List specials) throws Exception { + + this.functionListeners = new ArrayList<>(); + this.variableListeners = new ArrayList<>(); + this.specialListeners = new ArrayList<>(); + this.variables = new HashMap<>(); + this.zkClient = createZookeeperClient(zookeeperUrl); + this.context = createContext(properties, this.zkClient); + this.functionResolver = functionResolver; + this.specials = specials; + } + + public DefaultStellarShellExecutor( + FunctionResolver functionResolver, + Properties properties, + Optional zookeeperUrl) throws Exception { + + this(functionResolver, properties, zookeeperUrl, defaultSpecials()); + } + + public DefaultStellarShellExecutor( + Properties properties, + Optional zookeeperUrl) throws Exception { + + this(StellarFunctions.FUNCTION_RESOLVER(), properties, zookeeperUrl); + } + + /** + * The default specials that will be made available, if none are specified otherwise. + * @return The default special commands. + */ + public static List defaultSpecials() { + return Arrays.asList( + new AssignmentCommand(), + new DocCommand(), + new QuitCommand(), + new Comment(), + new MagicListFunctions(), + new MagicListVariables(), + new MagicDefineGlobal(), + new MagicUndefineGlobal(), + new MagicListGlobals() + ); + } + + @Override + public void init() { + StellarFunctions.initialize(this.context); + + // notify listeners about the available specials + for(SpecialCommand command : specials) { + notifySpecialListeners(command); + } + + // notify listeners about the available functions + for(StellarFunctionInfo fn : functionResolver.getFunctionInfo()) { + notifyFunctionListeners(fn); + } + } + + /** + * Add a listener that will be notified when a function is defined. + * @param listener The listener to notify. + */ + @Override + public void addFunctionListener(FunctionDefinedListener listener) { + this.functionListeners.add(listener); + } + + /** + * Notify function listeners that a function has been defined. + * @param functionInfo The function that was defined. + */ + private void notifyFunctionListeners(StellarFunctionInfo functionInfo) { + for(FunctionDefinedListener listener : functionListeners) { + listener.whenFunctionDefined(functionInfo); + } + } + + /** + * Add a listener that will be notified when a variable is defined. + * @param listener The listener to notify. + */ + @Override + public void addVariableListener(VariableDefinedListener listener) { + this.variableListeners.add(listener); + } + + /** + * Notify variable listeners that a variable has been (re)defined. + * @param variableName The variable name. + * @param result The variable result. + */ + private void notifyVariableListeners(String variableName, VariableResult result) { + for(VariableDefinedListener listener : variableListeners) { + listener.whenVariableDefined(variableName, result); + } + } + + /** + * Add a listener that will be notified when a magic command is defined. + * @param listener The listener to notify. + */ + @Override + public void addSpecialListener(SpecialDefinedListener listener) { + this.specialListeners.add(listener); + } + + /** + * Notify listeners that a magic command has been defined. + * @param specialCommand The magic command. + */ + private void notifySpecialListeners(SpecialCommand specialCommand) { + for(SpecialDefinedListener listener : specialListeners) { + listener.whenSpecialDefined(specialCommand); + } + } + + @Override + public StellarResult execute(String expression) { + + // if only whitespace, there is nothing to do + expression = StringUtils.trimToEmpty(expression); + if(StringUtils.isBlank(expression)) { + return noop(); + } + + // is this a special command? + for(SpecialCommand command : specials) { + if(command.getMatcher().apply(expression)) { + return command.execute(expression, this); + } + } + + // otherwise, this must be a stellar expression + return executeStellar(expression); + } + + /** + * Retrieves the GLOBAL_CONFIG, if it exists. If it does not, it creates the GLOBAL_CONFIG + * and adds it to the Stellar execution context. + * + * @return The global configuration. + */ + @Override + public Map getGlobalConfig() { + Map globals; + Optional capability = getContext().getCapability(GLOBAL_CONFIG, false); + if (capability.isPresent()) { + globals = (Map) capability.get(); + + } else { + // if it does not exist, create it. this creates the global config for the current stellar executor + // session only. this does not change the global config maintained externally in zookeeper + globals = new HashMap<>(); + getContext().addCapability(GLOBAL_CONFIG, () -> globals); + } + + return globals; + } + + @Override + public void assign(String variableName, Object value, Optional expression) { + + // perform the variable assignment + VariableResult varResult = VariableResult.withExpression(value, expression); + this.variables.put(variableName, varResult); + + // notify any listeners + notifyVariableListeners(variableName, varResult); + } + + @Override + public FunctionResolver getFunctionResolver() { + return functionResolver; + } + + @Override + public Map getState() { + return UnmodifiableMap.decorate(variables); + } + + /** + * Returns all variables that have been defined. Unlike 'getState' this unwraps + * the VariableResult so that we have the actual value. + * + * @return All variables that have been defined. + */ + public Map getVariables() { + return Maps.transformValues(variables, (v) -> v.getResult()); + } + + @Override + public Context getContext() { + return context; + } + + /** + * Creates a Zookeeper client. + * @param zookeeperUrl The Zookeeper URL. + */ + private Optional createZookeeperClient(Optional zookeeperUrl) { + Optional client = Optional.empty(); + + // can only create client, if have valid zookeeper URL + if(zookeeperUrl.isPresent()) { + String url = zookeeperUrl.get(); + if (StringUtils.isNotBlank(url)) { + + LOG.debug(String.format("Connecting to Zookeeper; url=%s", url)); + CuratorFramework c = ConfigurationsUtils.getClient(url); + c.start(); + client = Optional.of(c); + } + } + + return client; + } + + /** + * Creates a Context initialized with configuration stored in Zookeeper. + * @param properties Properties to configure the context. + * @param zkClient An optional Zookeeper client. + */ + private Context createContext(Properties properties, Optional zkClient) throws Exception { + + Context.Builder contextBuilder = new Context.Builder() + .with(SHELL_VARIABLES, () -> variables) + .with(STELLAR_CONFIG, () -> properties); + + // load global configuration from zookeeper + if (zkClient.isPresent()) { + Map global = fetchGlobalConfig(zkClient.get()); + contextBuilder + .with(GLOBAL_CONFIG, () -> global) + .with(ZOOKEEPER_CLIENT, () -> zkClient.get()) + .with(STELLAR_CONFIG, () -> getStellarConfig(global, properties)); + } + + return contextBuilder.build(); + } + + /** + * Fetches the global configuration from Zookeeper. + * @param zkClient The Zookeeper client. + * @return The global configuration retrieved from Zookeeper. + * @throws Exception + */ + private Map fetchGlobalConfig(CuratorFramework zkClient) throws Exception { + byte[] raw = readGlobalConfigBytesFromZookeeper(zkClient); + return JSONUtils.INSTANCE.load( + new ByteArrayInputStream(raw), + new TypeReference>() {}); + } + + /** + * @param globalConfig The global configuration. + * @param props Property values + * @return The Stellar configuration. + */ + private Map getStellarConfig(Map globalConfig, Properties props) { + Map stellarConfig = new HashMap<>(); + stellarConfig.putAll(globalConfig); + if(props != null) { + for (Map.Entry kv : props.entrySet()) { + stellarConfig.put(kv.getKey().toString(), kv.getValue()); + } + } + return stellarConfig; + } + + /** + * Executes Stellar expressions. + * @param expression The expression to execute. + */ + private StellarResult executeStellar(String expression) { + StellarResult result; + + try { + // execute the stellar expression + VariableResolver variableResolver = new MapVariableResolver(getVariables()); + Object exprResult = new StellarProcessor().parse(expression, variableResolver, functionResolver, context); + result = success(exprResult); + + } catch (Throwable t) { + result = error(t); + } + + return result; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java new file mode 100644 index 0000000000..35ab5ffbea --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java @@ -0,0 +1,45 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +/** + * Provides auto-complete functionality for Stellar. + */ +public interface StellarAutoCompleter { + + /** + * Auto-completes based on the given buffer. + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + Iterable autoComplete(String buffer); + + /** + * Adds a candidate for auto-completing function names. + * @param name The name of the function candidate. + */ + void addCandidateFunction(String name); + + /** + * Adds a candidate for auto-completing variable names. + * @param name The name of the function candidate. + */ + void addCandidateVariable(String name); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java new file mode 100644 index 0000000000..9734241b27 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java @@ -0,0 +1,51 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import org.apache.metron.stellar.common.shell.specials.SpecialCommand; +import org.apache.metron.stellar.dsl.StellarFunctionInfo; + +/** + * A listener will be notified about events that occur during the + * execution of Stellar expressions. + */ +public class StellarExecutionListeners { + + /** + * A listener that is notified when a function is defined. + */ + public interface FunctionDefinedListener { + void whenFunctionDefined(StellarFunctionInfo functionInfo); + } + + /** + * A listener that is notified when a variable is defined or redefined. + */ + public interface VariableDefinedListener { + void whenVariableDefined(String variableName, VariableResult result); + } + + /** + * A listener that is notified when a special command is defined. + */ + public interface SpecialDefinedListener { + void whenSpecialDefined(SpecialCommand magic); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java new file mode 100644 index 0000000000..f4ebf2ef73 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java @@ -0,0 +1,44 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +/** + * Notifies listeners when events occur during the execution of Stellar expressions. + */ +public interface StellarExecutionNotifier { + + /** + * Add a listener that will be notified when a magic command is defined. + * @param listener The listener to notify. + */ + void addSpecialListener(StellarExecutionListeners.SpecialDefinedListener listener); + + /** + * Add a listener that will be notified when a function is defined. + * @param listener The listener to notify. + */ + void addFunctionListener(StellarExecutionListeners.FunctionDefinedListener listener); + + /** + * Add a listener that will be notified when a variable is defined. + * @param listener The listener to notify. + */ + void addVariableListener(StellarExecutionListeners.VariableDefinedListener listener); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java deleted file mode 100644 index 1be38c3673..0000000000 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.metron.stellar.common.shell; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import java.io.ByteArrayInputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.SortedMap; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.commons.collections4.trie.PatriciaTrie; -import org.apache.commons.lang.StringUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.configuration.ConfigurationsUtils; -import org.apache.metron.stellar.common.utils.JSONUtils; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.MapVariableResolver; -import org.apache.metron.stellar.dsl.StellarFunctionInfo; -import org.apache.metron.stellar.dsl.StellarFunctions; -import org.apache.metron.stellar.dsl.VariableResolver; -import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import org.jboss.aesh.console.Console; - -import static org.apache.metron.stellar.common.configuration.ConfigurationsUtils.readGlobalConfigBytesFromZookeeper; -import static org.apache.metron.stellar.common.shell.StellarExecutor.OperationType.DOC; -import static org.apache.metron.stellar.common.shell.StellarExecutor.OperationType.NORMAL; -import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; -import static org.apache.metron.stellar.dsl.Context.Capabilities.STELLAR_CONFIG; -import static org.apache.metron.stellar.dsl.Context.Capabilities.ZOOKEEPER_CLIENT; - -/** - * Executes Stellar expressions and maintains state across multiple invocations. - */ -public class StellarExecutor { - - public static String SHELL_VARIABLES = "shellVariables"; - public static String CONSOLE = "console"; - - private ReadWriteLock indexLock = new ReentrantReadWriteLock(); - - public static class VariableResult { - private String expression; - private Object result; - - public VariableResult(String expression, Object result) { - this.expression = expression; - this.result = result; - } - - public String getExpression() { - return expression; - } - - public Object getResult() { - return result; - } - - @Override - public String toString() { - String ret = "" + result; - if(expression != null) { - ret += " via " + expression; - } - return ret; - } - } - - /** - * Prefix tree index of auto-completes. - */ - private PatriciaTrie autocompleteIndex; - - /** - * The variables known by Stellar. - */ - private Map variables; - - /** - * The function resolver. - */ - private FunctionResolver functionResolver; - - /** - * A Zookeeper client. Only defined if given a valid Zookeeper URL. - */ - private Optional client; - - /** - * The Stellar execution context. - */ - private Context context; - - private Console console; - - public enum OperationType { - DOC - , MAGIC - , NORMAL - } - - public interface AutoCompleteTransformation { - String transform(OperationType type, String key); - } - - public enum AutoCompleteType implements AutoCompleteTransformation{ - FUNCTION((type, key) -> { - if(type == DOC) { - return StellarShell.DOC_PREFIX + key; - } - else if(type == NORMAL) { - return key + "("; - } - return key; - }) - , VARIABLE((type, key) -> key ) - , TOKEN((type, key) -> key) - ; - - AutoCompleteTransformation transform; - AutoCompleteType(AutoCompleteTransformation transform) { - this.transform = transform; - } - - @Override - public String transform(OperationType type, String key) { - return transform.transform(type, key); - } - } - - /** - * @param console The console used to drive the REPL. - * @param properties The Stellar properties. - * @throws Exception - */ - public StellarExecutor(Console console, Properties properties) throws Exception { - this(null, console, properties); - } - - /** - * @param console The console used to drive the REPL. - * @param properties The Stellar properties. - * @throws Exception - */ - public StellarExecutor(String zookeeperUrl, Console console, Properties properties) throws Exception { - this.variables = new HashMap<>(); - this.client = createClient(zookeeperUrl); - this.context = createContext(properties); - - // initialize the default function resolver - StellarFunctions.initialize(this.context); - this.functionResolver = StellarFunctions.FUNCTION_RESOLVER(); - - this.autocompleteIndex = initializeIndex(); - this.console = console; - - // asynchronously update the index with function names found from a classpath scan. - new Thread( () -> { - Iterable functions = functionResolver.getFunctionInfo(); - indexLock.writeLock().lock(); - try { - for(StellarFunctionInfo info: functions) { - String functionName = info.getName(); - autocompleteIndex.put(functionName, AutoCompleteType.FUNCTION); - } - } finally { - System.out.println("Functions loaded, you may refer to functions now..."); - indexLock.writeLock().unlock(); - } - }).start(); - } - - private PatriciaTrie initializeIndex() { - Map index = new HashMap<>(); - - index.put("==", AutoCompleteType.TOKEN); - index.put(">=", AutoCompleteType.TOKEN); - index.put("<=", AutoCompleteType.TOKEN); - index.put(":=", AutoCompleteType.TOKEN); - index.put("quit", AutoCompleteType.TOKEN); - index.put(StellarShell.MAGIC_FUNCTIONS, AutoCompleteType.FUNCTION); - index.put(StellarShell.MAGIC_VARS, AutoCompleteType.FUNCTION); - index.put(StellarShell.MAGIC_GLOBALS, AutoCompleteType.FUNCTION); - index.put(StellarShell.MAGIC_DEFINE, AutoCompleteType.FUNCTION); - index.put(StellarShell.MAGIC_UNDEFINE, AutoCompleteType.FUNCTION); - return new PatriciaTrie<>(index); - } - - public Iterable autoComplete(String buffer, final OperationType opType) { - indexLock.readLock().lock(); - try { - SortedMap ret = autocompleteIndex.prefixMap(buffer); - if (ret.isEmpty()) { - return new ArrayList<>(); - } - return Iterables.transform(ret.entrySet(), kv -> kv.getValue().transform(opType, kv.getKey())); - } - finally { - indexLock.readLock().unlock(); - } - } - - /** - * Creates a Zookeeper client. - * @param zookeeperUrl The Zookeeper URL. - */ - private Optional createClient(String zookeeperUrl) { - - // can only create client, if have valid zookeeper URL - if(StringUtils.isNotBlank(zookeeperUrl)) { - CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl); - client.start(); - return Optional.of(client); - - } else { - return Optional.empty(); - } - } - - /** - * Creates a Context initialized with configuration stored in Zookeeper. - */ - private Context createContext(Properties properties) throws Exception { - - Context.Builder contextBuilder = new Context.Builder() - .with(SHELL_VARIABLES, () -> variables) - .with(CONSOLE, () -> console) - .with(STELLAR_CONFIG, () -> properties); - - // load global configuration from zookeeper - if (client.isPresent()) { - - // fetch the global configuration - Map global = JSONUtils.INSTANCE.load( - new ByteArrayInputStream(readGlobalConfigBytesFromZookeeper(client.get())), - new TypeReference>() {}); - - contextBuilder - .with(GLOBAL_CONFIG, () -> global) - .with(ZOOKEEPER_CLIENT, () -> client.get()) - .with(STELLAR_CONFIG, () -> getStellarConfig(global, properties)); - } - - return contextBuilder.build(); - } - - private Map getStellarConfig(Map globalConfig, Properties props) { - Map ret = new HashMap<>(); - ret.putAll(globalConfig); - if(props != null) { - for (Map.Entry kv : props.entrySet()) { - ret.put(kv.getKey().toString(), kv.getValue()); - } - } - return ret; - } - - /** - * Executes the Stellar expression and returns the result. - * @param expression The Stellar expression to execute. - * @return The result of the expression. - */ - public Object execute(String expression) { - VariableResolver variableResolver = new MapVariableResolver(Maps.transformValues(variables, result -> result.getResult()) - , Collections.emptyMap()); - StellarProcessor processor = new StellarProcessor(); - return processor.parse(expression, variableResolver, functionResolver, context); - } - - /** - * Assigns a value to a variable. - * @param variable The name of the variable. - * @param value The value of the variable - */ - public void assign(String variable, String expression, Object value) { - this.variables.put(variable, new VariableResult(expression, value)); - indexLock.writeLock().lock(); - try { - if (value != null) { - this.autocompleteIndex.put(variable, AutoCompleteType.VARIABLE); - } else { - this.autocompleteIndex.remove(variable); - } - } - finally { - indexLock.writeLock().unlock(); - } - } - - public Map getVariables() { - return this.variables; - } - - public FunctionResolver getFunctionResolver() { - return functionResolver; - } - - public Context getContext() { - return context; - } -} - diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarResult.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarResult.java new file mode 100644 index 0000000000..4e5c81ee90 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarResult.java @@ -0,0 +1,195 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import java.util.Optional; + +/** + * The result of executing a Stellar expression within a StellarShellExecutor. + */ +public class StellarResult { + + /** + * Indicates that a Stellar expression resulted in either + * success or an error. + */ + enum Status { + SUCCESS, + ERROR, + TERMINATE + } + + /** + * Indicates either success or failure of executing the expression. + */ + private Status status; + + /** + * The result of executing the expression. Only valid when execution is successful. + */ + private Optional value; + + /** + * The error that occurred when executing the expression. Only valid when execution results in an error. + */ + private Optional exception; + + /** + * Indicates if the value is null; + * + * A null is a valid result, but cannot be unwrapped from an Optional. Because of this + * a boolean is used to indicate if the result is a success and the value is null. + */ + private boolean isValueNull; + + /** + * Private constructor to construct a result indicate success. Use the static methods; success. + * + * @param status Indicates success or failure. + * @param value The value of executing the expression. + */ + private StellarResult(Status status, Object value) { + this.status = status; + this.value = Optional.ofNullable(value); + this.exception = Optional.empty(); + this.isValueNull = (value == null) && (status == Status.SUCCESS); + } + + /** + * Private constructor to construct a result indicating an error occurred. Use the static method; error. + * + * @param status Indicates success or failure. + * @param exception The exception that occurred when executing the expression. + */ + private StellarResult(Status status, Throwable exception) { + this.status = status; + this.value = Optional.empty(); + this.exception = Optional.of(exception); + this.isValueNull = false; + } + + /** + * Create a result indicating the execution of an expression was successful. + * + * @param value The result of executing the expression. + * @return A Result indicating success. + */ + public static StellarResult success(Object value) { + return new StellarResult(Status.SUCCESS, value); + } + + /** + * Create a result indicating that the execution of an expression was not successful. + * + * @param exception The exception that occurred while executing the expression. + * @return A Result indicating that an error occurred. + */ + public static StellarResult error(Throwable exception) { + return new StellarResult(Status.ERROR, exception); + } + + /** + * Create a result indicating that the execution of an expression was not successful. + * + * @param errorMessage An error message. + * @return A Result indicating that an error occurred. + */ + public static StellarResult error(String errorMessage) { + return new StellarResult(Status.ERROR, new IllegalArgumentException(errorMessage)); + } + + /** + * Indicates an empty result; one that is successful yet has no result. For example, + * executing a comment. + * + * @return An empty result. + */ + public static StellarResult noop() { + return new StellarResult(Status.SUCCESS, ""); + } + + /** + * Indicates that the user would like to terminate the session. + * + * @return A result indicating that the session should be terminated. + */ + public static StellarResult terminate() { + return new StellarResult(Status.TERMINATE, ""); + } + + /** + * @return True, if the result indicates success. Otherwise, false. + */ + public boolean isSuccess() { + return status == Status.SUCCESS; + } + + /** + * @return True, if the result indicates an error. Otherwise, false. + */ + public boolean isError() { + return status == Status.ERROR; + } + + /** + * @return True, if status indicates terminate was requested. Otherwise, false. + */ + public boolean isTerminate() { + return status == Status.TERMINATE; + } + + /** + * @return True, if the value is null. Otherwise, false. + */ + public boolean isValueNull() { + return isValueNull; + } + + /** + * @return The status which indicates success or failure. + */ + public Status getStatus() { + return status; + } + + /** + * @return An optional value that only applies when status is success. + */ + public Optional getValue() { + return value; + } + + /** + * @return An optional exception that only applies when status is error. + */ + public Optional getException() { + return exception; + } + + @Override + public String toString() { + return "StellarResult{" + + "status=" + status + + ", value=" + value + + ", exception=" + exception + + ", isValueNull=" + isValueNull + + '}'; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java deleted file mode 100644 index 6f23a5ebc4..0000000000 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ /dev/null @@ -1,557 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.metron.stellar.common.shell; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import org.apache.commons.cli.*; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.PropertyConfigurator; -import org.apache.metron.stellar.common.StellarAssignment; -import org.apache.metron.stellar.common.utils.JSONUtils; -import org.apache.metron.stellar.dsl.StellarFunctionInfo; -import org.jboss.aesh.complete.CompleteOperation; -import org.jboss.aesh.complete.Completion; -import org.jboss.aesh.console.AeshConsoleCallback; -import org.jboss.aesh.console.Console; -import org.jboss.aesh.console.ConsoleOperation; -import org.jboss.aesh.console.Prompt; -import org.jboss.aesh.console.settings.SettingsBuilder; -import org.jboss.aesh.terminal.CharacterType; -import org.jboss.aesh.terminal.Color; -import org.jboss.aesh.terminal.TerminalCharacter; -import org.jboss.aesh.terminal.TerminalColor; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; - -/** - * A REPL environment for Stellar. - * - * Useful for debugging Stellar expressions. - */ -public class StellarShell extends AeshConsoleCallback implements Completion { - - private static final String WELCOME = "Stellar, Go!\n" + - "Please note that functions are loading lazily in the background and will be unavailable until loaded fully."; - private List EXPRESSION_PROMPT = new ArrayList() - {{ - add(new TerminalCharacter('[', new TerminalColor(Color.RED, Color.DEFAULT))); - add(new TerminalCharacter('S', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('t', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('e', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('a', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('r', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter(']', new TerminalColor(Color.RED, Color.DEFAULT))); - add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); - add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); - add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); - add(new TerminalCharacter(' ', new TerminalColor(Color.DEFAULT, Color.DEFAULT))); - }}; - - public static final String ERROR_PROMPT = "[!] "; - public static final String MAGIC_PREFIX = "%"; - public static final String MAGIC_FUNCTIONS = MAGIC_PREFIX + "functions"; - public static final String MAGIC_VARS = MAGIC_PREFIX + "vars"; - public static final String DOC_PREFIX = "?"; - public static final String STELLAR_PROPERTIES_FILENAME = "stellar.properties"; - public static final String MAGIC_GLOBALS = MAGIC_PREFIX + "globals"; - public static final String MAGIC_DEFINE = MAGIC_PREFIX + "define"; - public static final String MAGIC_UNDEFINE = MAGIC_PREFIX + "undefine"; - - private StellarExecutor executor; - - private Console console; - - /** - * Execute the Stellar REPL. - */ - public static void main(String[] args) throws Exception { - StellarShell shell = new StellarShell(args); - shell.execute(); - } - - /** - * Create a Stellar REPL. - * @param args The commmand-line arguments. - */ - public StellarShell(String[] args) throws Exception { - - // define valid command-line options - Options options = new Options(); - options.addOption("z", "zookeeper", true, "Zookeeper URL fragment in the form [HOSTNAME|IPADDRESS]:PORT"); - options.addOption("v", "variables", true, "File containing a JSON Map of variables"); - options.addOption("irc", "inputrc", true, "File containing the inputrc if not the default ~/.inputrc"); - options.addOption("na", "no_ansi", false, "Make the input prompt not use ANSI colors."); - options.addOption("h", "help", false, "Print help"); - options.addOption("p", "properties", true, "File containing Stellar properties"); - { - Option o = new Option("l", "log4j", true, "The log4j properties file to load"); - o.setArgName("FILE"); - o.setRequired(false); - options.addOption(o); - } - CommandLineParser parser = new PosixParser(); - CommandLine commandLine = parser.parse(options, args); - - // print help - if(commandLine.hasOption("h")) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("stellar", options); - System.exit(0); - } - - try { - StellarShellOptionsValidator.validateOptions(commandLine); - }catch(IllegalArgumentException e){ - System.out.println(e.getMessage()); - System.exit(1); - } - //setting up logging if specified - if(commandLine.hasOption("l")) { - PropertyConfigurator.configure(commandLine.getOptionValue("l")); - } - console = createConsole(commandLine); - executor = createExecutor(commandLine, console, getStellarProperties(commandLine)); - loadVariables(commandLine, executor); - console.setPrompt(new Prompt(EXPRESSION_PROMPT)); - console.addCompletion(this); - console.setConsoleCallback(this); - } - - /** - * Loads any variables defined in an external file. - * @param commandLine The command line arguments. - * @param executor The stellar executor. - * @throws IOException - */ - private static void loadVariables(CommandLine commandLine, StellarExecutor executor) throws IOException { - if(commandLine.hasOption("v")) { - - Map variables = JSONUtils.INSTANCE.load( - new File(commandLine.getOptionValue("v")), - new TypeReference>() {}); - - for(Map.Entry kv : variables.entrySet()) { - executor.assign(kv.getKey(), null, kv.getValue()); - } - } - } - - /** - * Creates the Stellar execution environment. - * @param commandLine The command line arguments. - * @param console The console which drives the REPL. - * @param properties Stellar properties. - */ - private static StellarExecutor createExecutor(CommandLine commandLine, Console console, Properties properties) throws Exception { - StellarExecutor executor; - - // create the executor - if(commandLine.hasOption("z")) { - String zookeeperUrl = commandLine.getOptionValue("z"); - executor = new StellarExecutor(zookeeperUrl, console, properties); - - } else { - executor = new StellarExecutor(console, properties); - } - - return executor; - } - - /** - * Creates the REPL's console. - * @param commandLine The command line options. - */ - private Console createConsole(CommandLine commandLine) { - - // console settings - boolean useAnsi = !commandLine.hasOption("na"); - SettingsBuilder settings = new SettingsBuilder().enableAlias(true) - .enableMan(true) - .ansi(useAnsi) - .parseOperators(false) - .inputStream(PausableInput.INSTANCE); - - if(commandLine.hasOption("irc")) { - settings = settings.inputrc(new File(commandLine.getOptionValue("irc"))); - } - - return new Console(settings.create()); - } - - /** - * Retrieves the Stellar properties. The properties are either loaded from a file in - * the classpath or a set of defaults are used. - */ - private Properties getStellarProperties(CommandLine commandLine) throws IOException { - Properties properties = new Properties(); - - if (commandLine.hasOption("p")) { - - // first attempt to load properties from a file specified on the command-line - try (InputStream in = new FileInputStream(commandLine.getOptionValue("p"))) { - if(in != null) { - properties.load(in); - } - } - - } else { - - // otherwise attempt to load properties from the classpath - try (InputStream in = getClass().getClassLoader().getResourceAsStream(STELLAR_PROPERTIES_FILENAME)) { - if(in != null) { - properties.load(in); - } - } - } - - return properties; - } - - /** - * Handles the main loop for the REPL. - */ - public void execute() { - - // welcome message and print globals - writeLine(WELCOME); - executor.getContext() - .getCapability(GLOBAL_CONFIG, false) - .ifPresent(conf -> writeLine(conf.toString())); - - console.start(); - } - - /** - * Handles user interaction when executing a Stellar expression. - * @param expression The expression to execute. - */ - private void handleStellar(String expression) { - - String stellarExpression = expression; - String variable = null; - if(StellarAssignment.isAssignment(expression)) { - StellarAssignment expr = StellarAssignment.from(expression); - variable = expr.getVariable(); - stellarExpression = expr.getStatement(); - } - else { - if (!stellarExpression.isEmpty()) { - stellarExpression = stellarExpression.trim(); - } - } - - try { - Object result = executor.execute(stellarExpression); - if (result != null && variable == null) { - writeLine(result.toString()); - } - if (variable != null) { - executor.assign(variable, stellarExpression, result); - } - } catch (Throwable t) { - if(variable != null) { - writeLine(String.format("%s ERROR: Variable %s not assigned", ERROR_PROMPT, variable)); - } - writeLine(ERROR_PROMPT + t.getMessage()); - t.printStackTrace(); - } - } - - /** - * Executes a magic expression. - * @param rawExpression The expression to execute. - */ - private void handleMagic(String rawExpression) { - - String[] expression = rawExpression.trim().split("\\s+"); - String command = expression[0]; - - if (MAGIC_FUNCTIONS.equals(command)) { - handleMagicFunctions(expression); - - } else if (MAGIC_VARS.equals(command)) { - handleMagicVars(); - - } else if (MAGIC_GLOBALS.equals(command)) { - handleMagicGlobals(); - - } else if (MAGIC_DEFINE.equals(command)) { - handleMagicDefine(rawExpression); - - } else if(MAGIC_UNDEFINE.equals(command)) { - handleMagicUndefine(expression); - - } else { - writeLine(ERROR_PROMPT + "undefined magic command: " + rawExpression); - } - } - - /** - * Handle a magic '%functions'. Lists all of the variables in-scope. - * @param expression - */ - private void handleMagicFunctions(String[] expression) { - - // if '%functions FOO' then show only functions that contain 'FOO' - Predicate nameFilter = (name -> true); - if (expression.length > 1) { - nameFilter = (name -> name.contains(expression[1])); - } - - // '%functions' -> list all functions in scope - String functions = StreamSupport - .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false) - .map(info -> String.format("%s", info.getName())) - .filter(nameFilter) - .sorted() - .collect(Collectors.joining(", ")); - writeLine(functions); - } - - /** - * Handle a magic '%vars'. Lists all of the variables in-scope. - */ - private void handleMagicVars() { - executor.getVariables() - .forEach((k, v) -> writeLine(String.format("%s = %s", k, v))); - } - - /** - * Handle a magic '%globals'. List all of the global configuration values. - */ - private void handleMagicGlobals() { - Map globals = getOrCreateGlobalConfig(executor); - writeLine(globals.toString()); - } - - /** - * Handle a magic '%define var=value'. Alter the global configuration. - * @param expression The expression passed to %define - */ - public void handleMagicDefine(String expression) { - - // grab the expression in '%define ' - String assignExpr = StringUtils.trimToEmpty(expression.substring(MAGIC_DEFINE.length())); - if (assignExpr.length() > 0) { - - // the expression must be an assignment - if(StellarAssignment.isAssignment(assignExpr)) { - StellarAssignment expr = StellarAssignment.from(assignExpr); - - // execute the expression - Object result = executor.execute(expr.getStatement()); - if (result != null) { - writeLine(result.toString()); - - // alter the global configuration - getOrCreateGlobalConfig(executor).put(expr.getVariable(), result); - } - - } else { - // the expression is not an assignment. boo! - writeLine(ERROR_PROMPT + MAGIC_DEFINE + " expected assignment expression"); - } - } - } - - /** - * Handle a magic '%undefine var'. Removes a variable from the global configuration. - * @param expression - */ - private void handleMagicUndefine(String[] expression) { - if(expression.length > 1) { - Map globals = getOrCreateGlobalConfig(executor); - globals.remove(expression[1]); - } - } - - /** - * Retrieves the GLOBAL_CONFIG, if it exists. If it does not, it creates the GLOBAL_CONFIG - * and adds it to the Stellar execution context. - * @param executor The Stellar executor. - * @return The global configuration. - */ - private Map getOrCreateGlobalConfig(StellarExecutor executor) { - Map globals; - Optional capability = executor.getContext().getCapability(GLOBAL_CONFIG, false); - if (capability.isPresent()) { - globals = (Map) capability.get(); - - } else { - // if it does not exist, create it. this creates the global config for the current stellar executor - // session only. this does not change the global config maintained externally in zookeeper - globals = new HashMap<>(); - executor.getContext().addCapability(GLOBAL_CONFIG, () -> globals); - } - return globals; - } - - /** - * Executes a doc expression. - * @param expression The doc expression to execute. - */ - private void handleDoc(String expression) { - - String functionName = StringUtils.substring(expression, 1); - StreamSupport - .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false) - .filter(info -> StringUtils.equals(functionName, info.getName())) - .map(info -> format(info)) - .forEach(doc -> write(doc)); - } - - /** - * Executes a quit. - */ - private void handleQuit() { - try { - console.stop(); - } catch (Throwable e) { - e.printStackTrace(); - } - } - - /** - * Formats the Stellar function info object into a readable string. - * @param info The stellar function info object. - * @return A readable string. - */ - private String format(StellarFunctionInfo info) { - StringBuffer ret = new StringBuffer(); - ret.append(info.getName() + "\n"); - ret.append(String.format("Description: %-60s\n\n", info.getDescription())); - if(info.getParams().length > 0) { - ret.append("Arguments:\n"); - for(String param : info.getParams()) { - ret.append(String.format("\t%-60s\n", param)); - } - ret.append("\n"); - } - ret.append(String.format("Returns: %-60s\n", info.getReturns())); - - return ret.toString(); - } - - /** - * Is a given expression a built-in magic? - * @param expression The expression. - */ - private boolean isMagic(String expression) { - return StringUtils.startsWith(expression, MAGIC_PREFIX); - } - - /** - * Is a given expression asking for function documentation? - * @param expression The expression. - */ - private boolean isDoc(String expression) { - return StringUtils.startsWith(expression, DOC_PREFIX); - } - - private void write(String out) { - System.out.print(out); - } - - private void writeLine(String out) { - console.getShell().out().println(out); - } - - @Override - public int execute(ConsoleOperation output) throws InterruptedException { - String expression = output.getBuffer().trim(); - if(StringUtils.isNotBlank(expression) ) { - if(isMagic(expression)) { - handleMagic( expression); - - } else if(isDoc(expression)) { - handleDoc(expression); - - } else if (expression.equals("quit")) { - handleQuit(); - - } else if(expression.charAt(0) == '#') { - return 0; // comment, do nothing - - } else { - handleStellar(expression); - } - } - - return 0; - } - - @Override - public void complete(CompleteOperation completeOperation) { - if(!completeOperation.getBuffer().isEmpty()) { - String lastToken = Iterables.getLast(Splitter.on(" ").split(completeOperation.getBuffer()), null); - if(lastToken != null && !lastToken.isEmpty()) { - lastToken = lastToken.trim(); - final String lastBit = lastToken; - final boolean isDocRequest = isDoc(lastToken); - if(isDocRequest) { - lastToken = lastToken.substring(1); - } - StellarExecutor.OperationType opType = StellarExecutor.OperationType.NORMAL; - if(isDocRequest) { - opType = StellarExecutor.OperationType.DOC; - } - else if(isMagic(lastToken)) { - opType = StellarExecutor.OperationType.MAGIC; - } - Iterable candidates = executor.autoComplete(lastToken, opType); - if(candidates != null && !Iterables.isEmpty(candidates)) { - completeOperation.setCompletionCandidates( Lists.newArrayList( - Iterables.transform(candidates, s -> stripOff(completeOperation.getBuffer(), lastBit) + s ) - ) - ); - } - } - } - } - - private static String stripOff(String baseString, String lastBit) { - int index = baseString.lastIndexOf(lastBit); - if(index < 0) { - return baseString; - } - return baseString.substring(0, index); - } -} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShellExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShellExecutor.java new file mode 100644 index 0000000000..7e2d4fc2bf --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShellExecutor.java @@ -0,0 +1,79 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; + +import java.util.Map; +import java.util.Optional; + +/** + * Responsible for executing Stellar in a shell-like environment. + * + * Provides the additional capabilities expected of executing Stellar + * in a shell-like environment including maintaining state, variable assignment, + * magic commands, doc strings, and comments. + */ +public interface StellarShellExecutor extends StellarExecutionNotifier { + + /** + * Initialize the Stellar executor. + */ + void init(); + + /** + * Execute the Stellar expression. + * @param expression The Stellar expression to execute. + * @return The result of executing the Stellar expression. + */ + StellarResult execute(String expression); + + /** + * Update the state of the executor by assign a value to a variable. + * @param variable The name of the variable. + * @param value The value to assign. + * @param expression The expression that resulted in the given value. Optional. + */ + void assign(String variable, Object value, Optional expression); + + /** + * The current state of the Stellar execution environment. + */ + Map getState(); + + /** + * Returns the Context for the Stellar execution environment. + * @return The execution context. + */ + Context getContext(); + + /** + * Returns the global configuration of the Stellar execution environment. + * @return A map of values defined in the global configuration. + */ + Map getGlobalConfig(); + + /** + * Returns the function resolver of the Stellar execution environment. + * @return The function resolver. + */ + FunctionResolver getFunctionResolver(); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/VariableResult.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/VariableResult.java new file mode 100644 index 0000000000..6cb1173b7a --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/VariableResult.java @@ -0,0 +1,101 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import java.util.Optional; + +/** + * The value assigned to a variable. + * + * Allows us to maintain not only the resulting value, but the + * expression that resulted in that value. + */ +public class VariableResult { + + /** + * The expression that resulted in the value. Not always available. + */ + private Optional expression; + + /** + * The value of the variable. + */ + private Object result; + + /** + * Create a new VariableResult when the expression that resulted in a value is known. + * + * @param value The value. + * @param expression The expression that resulted in the given value. + * @return A VariableResult. + */ + public static VariableResult withExpression(Object value, String expression) { + return new VariableResult(Optional.of(expression), value); + } + + /** + * Create a new VariableResult when the expression that resulted in a value is known. + * + * @param value The value. + * @param expression The expression that resulted in the given value. + * @return A VariableResult. + */ + public static VariableResult withExpression(Object value, Optional expression) { + return new VariableResult(expression, value); + } + + /** + * Create a new VariableResult when only the value is known. + * + * @param value The value. + * @return A VariableResult. + */ + public static VariableResult withValue(Object value) { + return new VariableResult(Optional.empty(), value); + } + + /** + * Private constructor. Use the static method 'withExpression' and 'withValue'. + * + * @param expression The expression that resulted in the given value. + * @param result The value assigned to the variable. + */ + private VariableResult(Optional expression, Object result) { + this.expression = expression; + this.result = result; + } + + public Optional getExpression() { + return expression; + } + + public Object getResult() { + return result; + } + + @Override + public String toString() { + String ret = "" + result; + if(getExpression().isPresent()) { + ret += " via " + expression.get(); + } + return ret; + } +} \ No newline at end of file diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java similarity index 99% rename from metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java rename to metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java index 2acd05895a..fad0115dea 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java @@ -17,7 +17,7 @@ * limitations under the License. * */ -package org.apache.metron.stellar.common.shell; +package org.apache.metron.stellar.common.shell.cli; import java.io.IOException; import java.io.InputStream; diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java new file mode 100644 index 0000000000..44ad28c347 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java @@ -0,0 +1,427 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.shell.cli; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.PropertyConfigurator; +import org.apache.metron.stellar.common.shell.DefaultStellarAutoCompleter; +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarAutoCompleter; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.utils.JSONUtils; +import org.jboss.aesh.complete.CompleteOperation; +import org.jboss.aesh.complete.Completion; +import org.jboss.aesh.console.AeshConsoleCallback; +import org.jboss.aesh.console.Console; +import org.jboss.aesh.console.ConsoleOperation; +import org.jboss.aesh.console.Prompt; +import org.jboss.aesh.console.settings.SettingsBuilder; +import org.jboss.aesh.terminal.CharacterType; +import org.jboss.aesh.terminal.Color; +import org.jboss.aesh.terminal.TerminalCharacter; +import org.jboss.aesh.terminal.TerminalColor; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.SHELL_VARIABLES; + +/** + * A REPL environment for Stellar. + * + * Useful for debugging Stellar expressions. + */ +public class StellarShell extends AeshConsoleCallback implements Completion { + + static final String WELCOME = "Stellar, Go!\n" + + "Functions are loading lazily in the background and will be unavailable until loaded fully."; + + private List EXPRESSION_PROMPT = new ArrayList() + {{ + add(new TerminalCharacter('[', new TerminalColor(Color.RED, Color.DEFAULT))); + add(new TerminalCharacter('S', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('t', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('e', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('a', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('r', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter(']', new TerminalColor(Color.RED, Color.DEFAULT))); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter(' ', new TerminalColor(Color.DEFAULT, Color.DEFAULT))); + }}; + + public static final String ERROR_PROMPT = "[!] "; + public static final String STELLAR_PROPERTIES_FILENAME = "stellar.properties"; + + /** + * Executes Stellar expressions for the shell environment. + */ + private StellarShellExecutor executor; + + /** + * The Aesh shell console. + */ + private Console console; + + /** + * Provides auto-complete functionality. + */ + private StellarAutoCompleter autoCompleter; + + /** + * Execute the Stellar REPL. + */ + public static void main(String[] args) throws Exception { + StellarShell shell = new StellarShell(args); + shell.run(); + } + + /** + * Create a Stellar REPL. + * @param args The commmand-line arguments. + */ + public StellarShell(String[] args) throws Exception { + + // define valid command-line options + CommandLineParser parser = new PosixParser(); + Options options = defineCommandLineOptions(); + CommandLine commandLine = parser.parse(options, args); + + // print help + if(commandLine.hasOption("h")) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("stellar", options); + System.exit(0); + } + + // validate the command line options + try { + StellarShellOptionsValidator.validateOptions(commandLine); + + } catch(IllegalArgumentException e){ + System.err.println(e.getMessage()); + System.exit(1); + } + + // setup logging, if specified + if(commandLine.hasOption("l")) { + PropertyConfigurator.configure(commandLine.getOptionValue("l")); + } + + console = createConsole(commandLine); + autoCompleter = new DefaultStellarAutoCompleter(); + Properties props = getStellarProperties(commandLine); + executor = createExecutor(commandLine, console, props, autoCompleter); + loadVariables(commandLine, executor); + console.setPrompt(new Prompt(EXPRESSION_PROMPT)); + console.addCompletion(this); + console.setConsoleCallback(this); + } + + /** + * @return The valid command line options. + */ + private Options defineCommandLineOptions() { + Options options = new Options(); + options.addOption( + "z", + "zookeeper", + true, + "Zookeeper URL fragment in the form [HOSTNAME|IPADDRESS]:PORT"); + options.addOption( + "v", + "variables", + true, + "File containing a JSON Map of variables"); + options.addOption( + "irc", + "inputrc", + true, + "File containing the inputrc if not the default ~/.inputrc"); + options.addOption( + "na", + "no_ansi", + false, + "Make the input prompt not use ANSI colors."); + options.addOption( + "h", + "help", + false, + "Print help"); + options.addOption( + "p", + "properties", + true, + "File containing Stellar properties"); + Option log4j = new Option( + "l", + "log4j", + true, + "The log4j properties file to load"); + log4j.setArgName("FILE"); + log4j.setRequired(false); + options.addOption(log4j); + + return options; + } + + /** + * Loads any variables defined in an external file. + * @param commandLine The command line arguments. + * @param executor The stellar executor. + * @throws IOException + */ + private static void loadVariables( + CommandLine commandLine, + StellarShellExecutor executor) throws IOException { + + if(commandLine.hasOption("v")) { + + // load variables defined in a file + String variablePath = commandLine.getOptionValue("v"); + Map variables = JSONUtils.INSTANCE.load( + new File(variablePath), + new TypeReference>() {}); + + // for each variable... + for(Map.Entry kv : variables.entrySet()) { + String variable = kv.getKey(); + Object value = kv.getValue(); + + // define the variable - no expression available + executor.assign(variable, value, Optional.empty()); + } + } + } + + /** + * Creates the Stellar execution environment. + * @param commandLine The command line arguments. + * @param console The console which drives the REPL. + * @param properties Stellar properties. + */ + private StellarShellExecutor createExecutor( + CommandLine commandLine, + Console console, + Properties properties, + StellarAutoCompleter autoCompleter) throws Exception { + + // setup zookeeper client + Optional zookeeperUrl = Optional.empty(); + if(commandLine.hasOption("z")) { + zookeeperUrl = Optional.of(commandLine.getOptionValue("z")); + } + + StellarShellExecutor executor = new DefaultStellarShellExecutor(properties, zookeeperUrl); + + // the 'CONSOLE' capability is only available with the CLI REPL + executor.getContext().addCapability(CONSOLE, () -> console); + + // allows some Stellar functions to access Stellar internals; should probably use %magics instead + executor.getContext().addCapability(SHELL_VARIABLES, () -> executor.getState()); + + // register the auto-completer to be notified when needed + executor.addSpecialListener( (special) -> autoCompleter.addCandidateFunction(special.getCommand())); + executor.addFunctionListener( (function) -> autoCompleter.addCandidateFunction(function.getName())); + executor.addVariableListener((name, val) -> autoCompleter.addCandidateVariable(name)); + + executor.init(); + return executor; + } + + /** + * Creates the REPL's console. + * @param commandLine The command line options. + */ + private Console createConsole(CommandLine commandLine) { + + // console settings + boolean useAnsi = !commandLine.hasOption("na"); + SettingsBuilder settings = new SettingsBuilder().enableAlias(true) + .enableMan(true) + .ansi(useAnsi) + .parseOperators(false) + .inputStream(PausableInput.INSTANCE); + + if(commandLine.hasOption("irc")) { + settings = settings.inputrc(new File(commandLine.getOptionValue("irc"))); + } + + return new Console(settings.create()); + } + + /** + * Retrieves the Stellar properties. The properties are either loaded from a file in + * the classpath or a set of defaults are used. + */ + private Properties getStellarProperties(CommandLine commandLine) throws IOException { + Properties properties = new Properties(); + + if (commandLine.hasOption("p")) { + // attempt to load properties from a file specified on the command-line + try (InputStream in = new FileInputStream(commandLine.getOptionValue("p"))) { + if(in != null) { + properties.load(in); + } + } + + } else { + // otherwise attempt to load properties from the classpath + try (InputStream in = getClass().getClassLoader().getResourceAsStream(STELLAR_PROPERTIES_FILENAME)) { + if(in != null) { + properties.load(in); + } + } + } + + return properties; + } + + /** + * Handles the main loop for the REPL. + */ + public void run() { + // welcome message + writeLine(WELCOME); + + // print the globals if we got 'em + executor.getContext() + .getCapability(GLOBAL_CONFIG, false) + .ifPresent(conf -> writeLine(conf.toString())); + + console.start(); + } + + /** + * Quits the console. + */ + private void handleQuit() { + try { + console.stop(); + } catch (Throwable e) { + e.printStackTrace(); + } + } + + private void writeLine(String out) { + console.getShell().out().println(out); + } + + @Override + public int execute(ConsoleOperation output) throws InterruptedException { + + // grab the user the input + String expression = StringUtils.trimToEmpty(output.getBuffer()); + if(StringUtils.isNotBlank(expression) ) { + + // execute the expression + StellarResult result = executor.execute(expression); + + if(result.isSuccess()) { + // on success + result.getValue().ifPresent(v -> writeLine(v.toString())); + + } else if (result.isError()) { + // on error + result.getException().ifPresent(e -> writeLine(ERROR_PROMPT + e.getMessage())); + result.getException().ifPresent(e -> e.printStackTrace()); + + } else if(result.isTerminate()) { + // on quit + handleQuit(); + + } else { + // should never happen + throw new IllegalStateException("An execution result is neither a success nor a failure. Please file a bug report."); + } + } + + return 0; + } + + /** + * Performs auto-completion for the shell. + * @param completeOperation The auto-complete operation. + */ + @Override + public void complete(CompleteOperation completeOperation) { + String buffer = completeOperation.getBuffer(); + final String lastToken = getLastToken(buffer); + Iterable candidates = autoCompleter.autoComplete(buffer); + + // transform the candidates into valid completions + if(candidates != null && !Iterables.isEmpty(candidates)) { + for(String candidate : candidates) { + String completion = stripOff(buffer, lastToken) + candidate; + completeOperation.addCompletionCandidate(completion); + } + } + } + + private static String getLastToken(String buffer) { + String lastToken = Iterables.getLast(Splitter.on(" ").split(buffer), null); + return lastToken.trim(); + } + + private static String stripOff(String baseString, String lastBit) { + int index = baseString.lastIndexOf(lastBit); + if(index < 0) { + return baseString; + } + return baseString.substring(0, index); + } + + /** + * @return The executor of Stellar expressions. + */ + public StellarShellExecutor getExecutor() { + return executor; + } + + /** + * @return The console. + */ + public Console getConsole() { + return console; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShellOptionsValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidator.java similarity index 98% rename from metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShellOptionsValidator.java rename to metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidator.java index ab92401d9a..585add5434 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShellOptionsValidator.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidator.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.stellar.common.shell; +package org.apache.metron.stellar.common.shell.cli; import java.io.File; import java.net.InetAddress; diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java new file mode 100644 index 0000000000..e253b3b10a --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.StellarAssignment; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; + +import java.util.Optional; +import java.util.function.Function; + +import static org.apache.metron.stellar.common.shell.StellarResult.error; + +/** + * A special command that allows for variable assignment. Variable + * assignment is not implemented directly within Stellar. + * + * x := 2 + 2 + */ +public class AssignmentCommand implements SpecialCommand { + + public static final String ASSIGNMENT_OP = ":="; + + @Override + public Function getMatcher() { + return (input) -> StellarAssignment.isAssignment(input); + } + + @Override + public String getCommand() { + return ASSIGNMENT_OP; + } + + /** + * Handles variable assignment. + * @param input The assignment expression to execute. + * @param executor A stellar execution environment. + * @return + */ + @Override + public StellarResult execute(String input, StellarShellExecutor executor) { + assert StellarAssignment.isAssignment(input); + + // extract the variable and assignment expression + StellarAssignment assignment = StellarAssignment.from(input); + String varName = assignment.getVariable(); + String varExpr = assignment.getStatement(); + + // execute the stellar expression + StellarResult result = executor.execute(varExpr); + if(result.isSuccess()) { + + Object value = null; + if(result.getValue().isPresent()) { + value = result.getValue().get(); + + } else if(result.isValueNull()) { + value = null; + } + + // variable assignment + executor.assign(varName, value, Optional.of(varExpr)); + return result; + + } else { + return error("Assignment expression failed"); + } + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/Comment.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/Comment.java new file mode 100644 index 0000000000..cea0c91a24 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/Comment.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; + +import java.util.function.Function; + +import static org.apache.commons.lang3.StringUtils.startsWith; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.apache.metron.stellar.common.shell.StellarResult.noop; + +/** + * A special command that handles comments. + * + * # this is a comment + */ +public class Comment implements SpecialCommand { + + public static final String COMMENT_PREFIX = "#"; + + @Override + public String getCommand() { + return "#"; + } + + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), COMMENT_PREFIX); + } + + @Override + public StellarResult execute(String expression, StellarShellExecutor executor) { + return noop(); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/DocCommand.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/DocCommand.java new file mode 100644 index 0000000000..f362f74587 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/DocCommand.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.dsl.StellarFunctionInfo; + +import java.util.Optional; +import java.util.Spliterator; +import java.util.function.Function; +import java.util.stream.StreamSupport; + +import static org.apache.metron.stellar.common.shell.StellarResult.error; +import static org.apache.metron.stellar.common.shell.StellarResult.success; + +/** + * A special command that allows a user to request doc string + * about a Stellar function. + * + * For example `?TO_STRING` will output the docs for the function `TO_STRING` + */ +public class DocCommand implements SpecialCommand { + + public static final String DOC_PREFIX = "?"; + + @Override + public String getCommand() { + return DOC_PREFIX; + } + + @Override + public Function getMatcher() { + return (input) -> StringUtils.startsWith(input, DOC_PREFIX); + } + + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + StellarResult result; + + // expect ?functionName + String functionName = StringUtils.substring(command, 1); + + // grab any docs for the given function + Spliterator fnIterator = executor.getFunctionResolver().getFunctionInfo().spliterator(); + Optional functionInfo = StreamSupport + .stream(fnIterator, false) + .filter(info -> StringUtils.equals(functionName, info.getName())) + .findFirst(); + + if(functionInfo.isPresent()) { + result = success(docFormat(functionInfo.get())); + } else { + result = error(String.format("No docs available for function '%s'", functionName)); + } + + return result; + } + + /** + * Formats the Stellar function info object into a readable string. + * @param info The stellar function info object. + * @return A readable string. + */ + private String docFormat(StellarFunctionInfo info) { + StringBuffer docString = new StringBuffer(); + + // name + docString.append(info.getName() + "\n"); + + // description + docString.append(String.format("Description: %-60s\n\n", info.getDescription())); + + // params + if(info.getParams().length > 0) { + docString.append("Arguments:\n"); + for(String param : info.getParams()) { + docString.append(String.format("\t%-60s\n", param)); + } + docString.append("\n"); + } + + // returns + docString.append(String.format("Returns: %-60s\n", info.getReturns())); + + return docString.toString(); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobal.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobal.java new file mode 100644 index 0000000000..6e094f950b --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobal.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.StellarAssignment; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; + +import java.util.function.Function; + +import static org.apache.commons.lang3.StringUtils.startsWith; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.apache.metron.stellar.common.shell.StellarResult.error; +import static org.apache.metron.stellar.common.shell.StellarResult.success; + +/** + * Allows a variable to be defined (or redefined) a within the global configuration. + * + * %define newVar := newValue + */ +public class MagicDefineGlobal implements SpecialCommand { + + public static final String MAGIC_DEFINE = "%define"; + + @Override + public String getCommand() { + return MAGIC_DEFINE; + } + + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_DEFINE); + } + + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + + // grab the expression in '%define ' + String assignExpr = StringUtils.trimToEmpty(command.substring(MAGIC_DEFINE.length())); + if(StringUtils.length(assignExpr) < 1) { + return error(MAGIC_DEFINE + " missing assignment expression"); + } + + // the expression must be an assignment + if(!StellarAssignment.isAssignment(assignExpr)) { + return error(MAGIC_DEFINE + " expected assignment expression"); + } + + // execute the expression + StellarAssignment expr = StellarAssignment.from(assignExpr); + StellarResult result = executor.execute(expr.getStatement()); + + // execution must be successful + if(!result.isSuccess()) { + return error(MAGIC_DEFINE + " expression execution failed"); + } + + // expression should have a result + if(!result.getValue().isPresent()) { + return error(MAGIC_DEFINE + " expression produced no result"); + } + + // alter the global configuration + Object value = result.getValue().get(); + executor.getGlobalConfig().put(expr.getVariable(), value); + + return success(value); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctions.java new file mode 100644 index 0000000000..22fc0021fb --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctions.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; + +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.apache.commons.lang3.StringUtils.*; + +/** + * A MagicCommand that lists the functions available within + * a Stellar execution environment. + * + * %functions + */ +public class MagicListFunctions implements SpecialCommand { + + public static final String MAGIC_FUNCTIONS = "%functions"; + + @Override + public String getCommand() { + return MAGIC_FUNCTIONS; + } + + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_FUNCTIONS); + } + + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + + // if '%functions FOO' then show only functions that contain 'FOO' + String startsWith = StringUtils.trimToEmpty(command.substring(MAGIC_FUNCTIONS.length())); + Predicate nameFilter = (name -> true); + if (StringUtils.isNotBlank(startsWith)) { + nameFilter = (name -> name.contains(startsWith)); + } + + // '%functions' -> list all functions in scope + String functions = StreamSupport + .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false) + .map(info -> String.format("%s", info.getName())) + .filter(nameFilter) + .sorted() + .collect(Collectors.joining(", ")); + + return StellarResult.success(functions); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobals.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobals.java new file mode 100644 index 0000000000..09a6679628 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobals.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; + +import java.util.Map; +import java.util.function.Function; + +import static org.apache.commons.lang3.StringUtils.startsWith; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; + +/** + * Displays all currently defined global configuration variables. + * + * %globals + */ +public class MagicListGlobals implements SpecialCommand { + + public static final String MAGIC_GLOBALS = "%globals"; + + @Override + public String getCommand() { + return MAGIC_GLOBALS; + } + + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_GLOBALS); + } + + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + Map globals = executor.getGlobalConfig(); + return StellarResult.success(globals.toString()); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListVariables.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListVariables.java new file mode 100644 index 0000000000..5669c44ac6 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListVariables.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.VariableResult; + +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.apache.commons.lang3.StringUtils.startsWith; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.apache.metron.stellar.common.shell.StellarResult.success; + +/** + * A MagicCommand that lists the variables available within + * the Stellar execution environment. + * + * %vars + */ +public class MagicListVariables implements SpecialCommand { + + public static final String MAGIC_VARS = "%vars"; + + @Override + public String getCommand() { + return MAGIC_VARS; + } + + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_VARS); + } + + /** + * Lists each variable, its value, and if available, the expression that resulted in that value. + * + * x = 4 via `2 + 2` + * + * @param command The command to execute. + * @param executor A stellar execution environment. + * @return + */ + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + + // format a string containing each variable and it's value + String vars = executor + .getState() + .entrySet() + .stream() + .map(e -> format(e)) + .collect(Collectors.joining(", ")); + + return success(vars); + } + + private String format(Map.Entry var) { + + // 'varName = varValue' + String out = String.format("%s = %s", var.getKey(), var.getValue().getResult()); + + // 'via varExpression', if the expression is known + if(var.getValue().getExpression().isPresent()) { + out += String.format(" via `%s`", var.getValue().getExpression().get()); + } + + return out; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobal.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobal.java new file mode 100644 index 0000000000..b7727e7dc4 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobal.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; + +import java.util.Map; +import java.util.function.Function; + +import static org.apache.commons.lang3.StringUtils.startsWith; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.apache.metron.stellar.common.shell.StellarResult.noop; +import static org.apache.metron.stellar.common.shell.StellarResult.error; + +/** + * Allows a variable to be removed from the global configuration. + * + * %undefine varName + */ +public class MagicUndefineGlobal implements SpecialCommand { + + public static final String MAGIC_UNDEFINE = "%undefine"; + + @Override + public String getCommand() { + return MAGIC_UNDEFINE; + } + + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_UNDEFINE); + } + + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + StellarResult result; + + String variable = StringUtils.trimToEmpty(command.substring(MAGIC_UNDEFINE.length())); + if(StringUtils.isNotBlank(variable)) { + + // remove the variable from the globals + Map globals = executor.getGlobalConfig(); + globals.remove(variable); + result = noop(); + + } else { + result = error(String.format("%s expected name of global, got '%s'", MAGIC_UNDEFINE, variable)); + } + + return result; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/QuitCommand.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/QuitCommand.java new file mode 100644 index 0000000000..7786d4d229 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/QuitCommand.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; + +import java.util.function.Function; + +import static org.apache.metron.stellar.common.shell.StellarResult.terminate; + +/** + * A special command that allows the user to 'quit' their REPL session. + * + * quit + */ +public class QuitCommand implements SpecialCommand { + + public static final String QUIT_COMMAND = "quit"; + + @Override + public String getCommand() { + return QUIT_COMMAND; + } + + @Override + public Function getMatcher() { + return (input) -> QUIT_COMMAND.equals(input); + } + + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + return terminate(); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/SpecialCommand.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/SpecialCommand.java new file mode 100644 index 0000000000..a40e1e53ec --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/SpecialCommand.java @@ -0,0 +1,60 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; + +import java.util.function.Function; + +/** + * A special command that can be run within a Stellar execution + * environment. + * + * Most functionality expected of running Stellar in a shell-like + * environment, that is not directly implemented in the Stellar + * language itself, is implemented as a SpecialCommand. This + * includes magics, doc strings, comments and quit. + * + * This is typically an action performed on the execution + * environment, not something that could be executed within Stellar. + */ +public interface SpecialCommand { + + /** + * @return A function that when applied to the input buffer returns + * true, if this special command should be applied. + */ + Function getMatcher(); + + /** + * @return The name of the command, used for auto-completion. + */ + String getCommand(); + + /** + * Execute the magic command. + * @param expression The expression to execute. + * @param executor A stellar execution environment. + * @return The result of executing the magic command. + */ + StellarResult execute(String expression, StellarShellExecutor executor); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Context.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Context.java index 3dbcc823f7..9568a05e3f 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Context.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Context.java @@ -34,6 +34,8 @@ public enum Capabilities { , ZOOKEEPER_CLIENT , SERVICE_DISCOVERER , STELLAR_CONFIG + , CONSOLE + , SHELL_VARIABLES } public enum ActivityType { @@ -72,16 +74,9 @@ public Context build() { } public static Context EMPTY_CONTEXT() { - return - new Context(new HashMap<>()){ - @Override - public Optional getCapability(String capability) { - return Optional.empty(); - } - }; + return new Context(new HashMap<>()){}; } - private Map capabilities; private Context( Map capabilities) { @@ -93,7 +88,6 @@ public Optional getCapability(Enum capability) { } public Optional getCapability(Enum capability, boolean errorIfNotThere) { - return getCapability(capability.toString(), errorIfNotThere); } diff --git a/metron-stellar/stellar-common/src/main/scripts/stellar b/metron-stellar/stellar-common/src/main/scripts/stellar index 2f1cdbe8d8..7b0f06d1b3 100644 --- a/metron-stellar/stellar-common/src/main/scripts/stellar +++ b/metron-stellar/stellar-common/src/main/scripts/stellar @@ -33,4 +33,4 @@ export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar) export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar) -java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.StellarShell "$@" +java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/StellarInterpreterTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/StellarCompilerTest.java similarity index 99% rename from metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/StellarInterpreterTest.java rename to metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/StellarCompilerTest.java index 36a7116182..f10ee5c80c 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/StellarInterpreterTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/StellarCompilerTest.java @@ -44,7 +44,7 @@ @RunWith(PowerMockRunner.class) @PrepareForTest({Deque.class, ArithmeticEvaluator.class, NumberLiteralEvaluator.class, ComparisonExpressionWithOperatorEvaluator.class}) -public class StellarInterpreterTest { +public class StellarCompilerTest { VariableResolver variableResolver; FunctionResolver functionResolver; Context context; diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleterTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleterTest.java new file mode 100644 index 0000000000..0d9d5e1f2a --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleterTest.java @@ -0,0 +1,190 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import com.google.common.collect.Lists; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.MatcherAssert.assertThat; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests the DefaultStellarAutoCompleter class. + */ +public class DefaultStellarAutoCompleterTest { + + DefaultStellarAutoCompleter completer; + + @Before + public void setup() { + completer = new DefaultStellarAutoCompleter(); + } + + @Test + public void testAutoCompleteFunction() { + + // setup some candidate functions for auto-completion + completer.addCandidateFunction("FREUD"); + completer.addCandidateFunction("FRIEND"); + completer.addCandidateFunction("FOE"); + + // need help auto-completer! + Iterable result = completer.autoComplete("FR"); + + // there are only 2 candidates that make any sense + List completes = Lists.newArrayList(result); + assertEquals(2, completes.size()); + + // these are functions and so should include an opening paren + assertThat(completes, hasItem("FREUD(")); + assertThat(completes, hasItem("FRIEND(")); + } + + @Test + public void testNoCandidateFunctions() { + + // setup some candidate functions for auto-completion + completer.addCandidateFunction("FREUD"); + completer.addCandidateFunction("FRIEND"); + completer.addCandidateFunction("FOE"); + + // need help auto-completer! + Iterable result = completer.autoComplete("G"); + + // there are no candidates that make any sense + List completes = Lists.newArrayList(result); + assertEquals(0, completes.size()); + } + + @Test + public void testAutoCompleteVariable() { + + // setup some candidates for auto-completion + completer.addCandidateVariable("very"); + completer.addCandidateVariable("vast"); + completer.addCandidateVariable("vat"); + + // need help auto-completer! + Iterable result = completer.autoComplete("va"); + + // there are only 2 candidates that make any sense + List completes = Lists.newArrayList(result); + assertEquals(2, completes.size()); + assertThat(completes, hasItem("vast")); + assertThat(completes, hasItem("vat")); + } + + @Test + public void testNoCandidateVariable() { + + // setup some candidates for auto-completion + completer.addCandidateVariable("very"); + completer.addCandidateVariable("vast"); + completer.addCandidateVariable("vat"); + + // need help auto-completer! + Iterable result = completer.autoComplete("x"); + + // there are only no candidates that make any sense + List completes = Lists.newArrayList(result); + assertEquals(0, completes.size()); + } + + @Test + public void testAutoCompleteDocString() { + + // setup some candidate functions for auto-completion + completer.addCandidateFunction("FREUD"); + completer.addCandidateFunction("FRIEND"); + completer.addCandidateFunction("FOE"); + + // need help auto-completer! + Iterable result = completer.autoComplete("?FR"); + + // there are only 2 candidates that make any sense + List completes = Lists.newArrayList(result); + assertEquals(2, completes.size()); + + // the suggestions should include the docstring prefix + assertThat(completes, hasItem("?FREUD")); + assertThat(completes, hasItem("?FRIEND")); + } + + @Test + public void testNoCandidateDocStrings() { + + // setup some candidate functions for auto-completion + completer.addCandidateFunction("FREUD"); + completer.addCandidateFunction("FRIEND"); + completer.addCandidateFunction("FOE"); + + // need help auto-completer! + Iterable result = completer.autoComplete("?G"); + + // there are only no candidates that make any sense + List completes = Lists.newArrayList(result); + assertEquals(0, completes.size()); + } + + @Test + public void testAutoCompleteMagic() { + + // setup some candidate functions for auto-completion + completer.addCandidateFunction("%vars"); + completer.addCandidateFunction("%vast"); + completer.addCandidateFunction("%verbotten"); + + // need help auto-completer! + Iterable result = completer.autoComplete("%va"); + + // there are only 2 candidates that make any sense + List completes = Lists.newArrayList(result); + assertEquals(2, completes.size()); + + // the suggestions should include the docstring prefix + assertThat(completes, hasItem("%vars")); + assertThat(completes, hasItem("%vast")); + } + + @Test + public void testNoCandidateMagic() { + + // setup some candidate functions for auto-completion + completer.addCandidateFunction("%vars"); + completer.addCandidateFunction("%vast"); + completer.addCandidateFunction("%verbotten"); + + // need help auto-completer! + Iterable result = completer.autoComplete("%xy"); + + // there are only no candidates that make any sense + List completes = Lists.newArrayList(result); + assertEquals(0, completes.size()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutorTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutorTest.java new file mode 100644 index 0000000000..ebba84f5d7 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutorTest.java @@ -0,0 +1,298 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests the DefaultStellarShellExecutor class. + */ +public class DefaultStellarShellExecutorTest { + + DefaultStellarShellExecutor executor; + boolean notified; + + @Before + public void setup() throws Exception { + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testAssignment() { + + // x = 2 + 2 + { + StellarResult result = executor.execute("x := 2 + 2"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(4, result.getValue().get()); + assertEquals(4, executor.getVariables().get("x")); + } + + // y = x + 2 + { + StellarResult result = executor.execute("y := x + 2"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(6, result.getValue().get()); + assertEquals(6, executor.getVariables().get("y")); + } + + // z = x + y + { + StellarResult result = executor.execute("z := x + y"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(10, result.getValue().get()); + assertEquals(10, executor.getVariables().get("z")); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testAssignmentOfLists() { + List expected = Arrays.asList(1,2,3,4,5); + + // assign a list to a variable + StellarResult result = executor.execute("x := [1,2,3,4,5]"); + + // the result should be a list + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(expected, result.getValue().get()); + + // the variable should also be the same list + List variable = (List) executor.getVariables().get("x"); + assertEquals(expected, variable); + } + + @Test + @SuppressWarnings("unchecked") + public void testAssignmentOfMaps() { + Map expected = ImmutableMap.builder() + .put("a", 10) + .put("b", 20) + .build(); + + // assign a list to a variable + StellarResult result = executor.execute("x := {'a':10, 'b':20}"); + + // the result should be a map + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(expected, result.getValue().get()); + + // the variable should also be the same list + Map variable = (Map) executor.getVariables().get("x"); + assertEquals(expected, variable); + } + + @Test + public void testAssignmentWithOddWhitespace() { + StellarResult result = executor.execute(" x := 2 + 2 "); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(4, result.getValue().get()); + assertEquals(4, executor.getVariables().get("x")); + } + + @Test + public void testBadAssignment() { + StellarResult result = executor.execute("x := 2 + "); + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + } + + @Test + public void testExpression() { + StellarResult result = executor.execute("2 + 2"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(4, result.getValue().get()); + } + + @Test + public void testExpressionWithOddWhitespace() { + StellarResult result = executor.execute(" 2 + 2"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(4, result.getValue().get()); + } + + @Test + public void testBadExpression() { + StellarResult result = executor.execute("2 + "); + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + } + + @Test + public void testMagicCommand() { + // create a var that we can see with the magic command + executor.execute("x := 2 + 2"); + + // just testing that we can execute the magic, not the actual result + StellarResult result = executor.execute("%vars"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertNotNull(result.getValue().get()); + } + + @Test + public void testDefineGlobal() { + + // create a global var + executor.execute("%define x := 2"); + + assertFalse(executor.getVariables().containsKey("x")); + + // the global should have been defined + assertEquals(2, executor.getGlobalConfig().get("x")); + } + + @Test + public void testBadMagicCommand() { + StellarResult result = executor.execute("%invalid"); + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + } + + @Test + public void testDocCommand() { + // just testing that we can execute the doc, not the actual result + StellarResult result = executor.execute("?TO_STRING"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertNotNull(result.getValue().get()); + } + + @Test + public void testBadDocCommand() { + StellarResult result = executor.execute("?INVALID"); + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + } + + @Test + public void testQuit() { + StellarResult result = executor.execute("quit"); + assertTrue(result.isTerminate()); + } + + @Test + public void testAssign() { + { + // this is how variables get loaded into the REPL directly from a file + executor.assign("x", 10, Optional.empty()); + } + { + StellarResult result = executor.execute("x + 2"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(12, result.getValue().get()); + } + } + + @Test + public void testNotifyVariableListeners() { + notified = false; + executor.addVariableListener((var, value) -> { + assertEquals("x", var); + assertEquals(4, value.getResult()); + notified = true; + }); + + executor.execute("x := 2 + 2"); + assertTrue(notified); + } + + @Test + public void testNotifySpecialListeners() throws Exception { + + // setup an executor + notified = false; + Properties props = new Properties(); + DefaultStellarShellExecutor executor = new DefaultStellarShellExecutor(props, Optional.empty()); + + // setup listener + notified = false; + executor.addSpecialListener((magic) -> { + assertNotNull(magic); + assertNotNull(magic.getCommand()); + notified = true; + }); + + // initialize... magics should be setup during initialization + executor.init(); + assertTrue(notified); + } + + @Test + public void testNotifyFunctionListeners() throws Exception { + // setup an executor + notified = false; + Properties props = new Properties(); + DefaultStellarShellExecutor executor = new DefaultStellarShellExecutor(props, Optional.empty()); + + // setup listener + notified = false; + executor.addFunctionListener((fn) -> { + assertNotNull(fn); + assertNotNull(fn.getName()); + assertNotNull(fn.getFunction()); + notified = true; + }); + + // initialize... magics should be setup during initialization + executor.init(); + assertTrue(notified); + } + + @Test + public void testEmptyInput() { + StellarResult result = executor.execute(""); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals("", result.getValue().get()); + } + + @Test + public void testComment() { + StellarResult result = executor.execute("# this is a comment"); + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals("", result.getValue().get()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/StellarResultTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/StellarResultTest.java new file mode 100644 index 0000000000..6431ee31af --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/StellarResultTest.java @@ -0,0 +1,165 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +public class StellarResultTest { + + /** + * Tests the 'success' method which is used to retrieve a StellarShellResult + * indicating success. + */ + @Test + public void testSuccess() { + final int expected = 2; + + // retrieve a result that indicates success + StellarResult result = StellarResult.success(expected); + assertNotNull(result); + + // validate the value + assertTrue(result.getValue().isPresent()); + assertEquals(expected, result.getValue().get()); + + // validate the exception + assertFalse(result.getException().isPresent()); + + // validate status + assertEquals(StellarResult.Status.SUCCESS, result.getStatus()); + assertTrue(result.isSuccess()); + assertFalse(result.isError()); + assertFalse(result.isTerminate()); + } + + /** + * Tests the 'error' method which is used to retrieve a StellarShellResult + * indicating that an error occurred. + */ + @Test + public void testError() { + final String expected = "my error message"; + + // retrieve a result that indicates success + StellarResult result = StellarResult.error(expected); + assertNotNull(result); + + // validate the value + assertFalse(result.getValue().isPresent()); + + // validate the exception + assertTrue(result.getException().isPresent()); + assertEquals(expected, result.getException().get().getMessage()); + + // validate status + assertEquals(StellarResult.Status.ERROR, result.getStatus()); + assertFalse(result.isSuccess()); + assertTrue(result.isError()); + assertFalse(result.isTerminate()); + } + + /** + * Tests the 'terminate' method which is used to retrieve a StellarShellResult + * indicating that a termination request was made. + */ + @Test + public void testTerminate() { + + // retrieve a result that indicates success + StellarResult result = StellarResult.terminate(); + assertNotNull(result); + + // validate the value + assertTrue(result.getValue().isPresent()); + + // validate the exception + assertFalse(result.getException().isPresent()); + + // validate status + assertEquals(StellarResult.Status.TERMINATE, result.getStatus()); + assertFalse(result.isSuccess()); + assertFalse(result.isError()); + assertTrue(result.isTerminate()); + } + + /** + * Tests the 'noop' method which is used to retrieve a StellarShellResult + * indicating that no operation occurred, nor was required. + */ + @Test + public void testNoop() { + + // retrieve a result that indicates success + StellarResult result = StellarResult.noop(); + assertNotNull(result); + + // validate the value + assertTrue(result.getValue().isPresent()); + + // validate the exception + assertFalse(result.getException().isPresent()); + + // validate status + assertEquals(StellarResult.Status.SUCCESS, result.getStatus()); + assertTrue(result.isSuccess()); + assertFalse(result.isError()); + assertFalse(result.isTerminate()); + } + + /** + * A success result where the value is null is perfectly acceptable. + */ + @Test + public void testSuccessWithNull() { + final Object expected = null; + + // retrieve a result that indicates success + StellarResult result = StellarResult.success(expected); + assertNotNull(result); + + // validate the value + assertTrue(result.isValueNull()); + + // validate the exception + assertFalse(result.getException().isPresent()); + + // validate status + assertEquals(StellarResult.Status.SUCCESS, result.getStatus()); + assertTrue(result.isSuccess()); + assertFalse(result.isError()); + assertFalse(result.isTerminate()); + } + + /** + * Tests the behavior of isValueNull() with error, noop and terminate conditions. + */ + @Test + public void testNonSuccessWithNull() { + assertFalse(StellarResult.error(new Exception()).isValueNull()); + assertFalse(StellarResult.error("error msg").isValueNull()); + assertFalse(StellarResult.noop().isValueNull()); + assertFalse(StellarResult.terminate().isValueNull()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/StellarShellOptionsValidatorTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidatorTest.java similarity index 98% rename from metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/StellarShellOptionsValidatorTest.java rename to metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidatorTest.java index 67a8ae7f63..8ca09599dc 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/StellarShellOptionsValidatorTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidatorTest.java @@ -18,7 +18,7 @@ * */ -package org.apache.metron.stellar.common.shell; +package org.apache.metron.stellar.common.shell.cli; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -28,9 +28,6 @@ import org.junit.Test; import java.io.File; -import java.util.regex.Pattern; - -import static org.junit.Assert.*; public class StellarShellOptionsValidatorTest { diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java new file mode 100644 index 0000000000..ef2475bee6 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common.shell.cli; + +import com.google.common.collect.Iterables; +import org.jboss.aesh.complete.CompleteOperation; +import org.jboss.aesh.console.AeshContext; +import org.jboss.aesh.console.ConsoleOperation; +import org.jboss.aesh.console.operator.ControlOperator; +import org.jboss.aesh.console.settings.DefaultAeshContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * Tests the StellarShell class. + */ +public class StellarShellTest { + + private StellarShell stellarShell; + private ByteArrayOutputStream out; + private ByteArrayOutputStream err; + + @Before + public void setup() throws Exception { + + out = new ByteArrayOutputStream(); + err = new ByteArrayOutputStream(); + + // setup streams so that we can capture stdout + System.setOut(new PrintStream(out)); + System.setErr(new PrintStream(err)); + + String[] args = new String[0]; + stellarShell = new StellarShell(args); + } + + @After + public void cleanUp() { + System.setOut(null); + System.setErr(null); + } + + /** + * @return The data written to stdout during the test (with newlines stripped out to simplify comparisons.) + */ + private String stdout() { + return out.toString().replace(System.lineSeparator(), ""); + } + + /** + * @return The data written to stdout during the test. + */ + private String stdoutWithNewlines() { + return out.toString(); + } + + + /** + * @return The data written to stderr during the test. + */ + private String stderr() { + return err.toString().replace(System.lineSeparator(), ""); + } + + /** + * @param buffer + * @return A ConsoleOperation that that StellarShell uses to drive input. + */ + private ConsoleOperation createOp(String buffer) { + return new ConsoleOperation(ControlOperator.APPEND_OUT, buffer); + } + + @Test + public void testExecuteStellar() throws Exception { + stellarShell.execute(createOp("2 + 2")); + assertEquals("4", stdout()); + } + + /** + * Ensure that Stellar lists are displayed correctly in the REPL. + */ + @Test + public void testExecuteWithStellarList() throws Exception { + stellarShell.execute(createOp("[1,2,3,4,5]")); + assertEquals("[1, 2, 3, 4, 5]", stdout()); + } + + /** + * Ensure that Stellar maps are displayed correctly in the REPL. + */ + @Test + public void testExecuteWithStellarMap() throws Exception { + stellarShell.execute(createOp("{ 'foo':2, 'key':'val' }")); + assertEquals("{foo=2, key=val}", stdout()); + } + + /** + * Ensure that 'bad' Stellar code is handled correctly by the REPL. + */ + @Test + public void testExecuteBadStellar() throws Exception { + stellarShell.execute(createOp("2 + ")); + final String expected = "[!] Unable to parse: 2 + "; + assertTrue(stdout().startsWith(expected)); + } + + /** + * The REPL should handle if no value is returned. Some Stellar expressions + * will result in no value. + */ + @Test + public void testExecuteNoop() throws Exception { + stellarShell.execute(createOp("x")); + assertEquals("", stdout()); + } + + /** + * The REPL should handle if the user chooses to quit. + */ + @Test + public void testQuit() throws Exception { + stellarShell.execute(createOp("quit")); + + // the console should not be running + assertFalse(stellarShell.getConsole().isRunning()); + } + + /** + * The REPL should handle if the user chooses to quit. + */ + @Test + public void testStart() throws Exception { + + StellarShell.main(new String[0]); + + // we should see the welcome prompt + assertTrue(stdoutWithNewlines().contains(StellarShell.WELCOME)); + } + + /** + * The REPL should support auto-completion. + */ + @Test + public void testAutoComplete() throws Exception { + + // the user's input that needs auto-completed + final String buffer = "TO_"; + + // the cursor is at the end of the buffer + int cursor = buffer.length(); + + // ask the shell to auto-complete + AeshContext context = new DefaultAeshContext(); + CompleteOperation op = new CompleteOperation(context, buffer, cursor); + stellarShell.complete(op); + + // we should have some auto-complete candidates + List candidates = op.getFormattedCompletionCandidates(); + assertTrue(candidates.size() > 0); + + // validate each candidate + for(String candidate: candidates) { + String completion = buffer + candidate; + + // the auto-complete should include an open paren + assertEquals("(", completion.substring(completion.length() - 1)); + + // the candidate should be a valid, defined function + String function = completion.substring(0, completion.length() - 1); + Iterable allFunctions = stellarShell.getExecutor().getFunctionResolver().getFunctions(); + String definedFunction = Iterables.find(allFunctions, (fn) -> fn.equals(function)); + assertEquals(function, definedFunction); + } + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java new file mode 100644 index 0000000000..899effb334 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java @@ -0,0 +1,212 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNull; + + +/** + * Tests the AssignmentCommand class. + */ +public class AssignmentCommandTest { + + AssignmentCommand command; + StellarShellExecutor executor; + + @Before + public void setup() throws Exception { + command = new AssignmentCommand(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals(":=", command.getCommand()); + } + + @Test + public void testShouldMatch() { + List inputs = Arrays.asList( + "x := 2 + 2", + " x := 2 + 2 ", + " x := 2", + " x := " + ); + for(String in : inputs) { + assertTrue("failed: " + in, command.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List inputs = Arrays.asList( + "2+2", + " %define x := 2", + "x" + ); + for(String in : inputs) { + assertFalse("failed: " + in, command.getMatcher().apply(in)); + } + } + + @Test + public void testAssignment() { + StellarResult result = command.execute("x := 2 + 2", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(4, result.getValue().get()); + + // validate assignment + assertEquals(4, executor.getState().get("x").getResult()); + } + + @Test + public void testAssignments() { + + // execute a series of assignments + assertTrue(command.execute("x := 2 + 2", executor).isSuccess()); + assertTrue(command.execute("y := 2 + x", executor).isSuccess()); + assertTrue(command.execute("z := x + y", executor).isSuccess()); + + // validate assignment + assertEquals(4, executor.getState().get("x").getResult()); + assertEquals(6, executor.getState().get("y").getResult()); + assertEquals(10, executor.getState().get("z").getResult()); + } + + @Test + public void testReassignment() { + + // execute a series of assignments + assertTrue(command.execute("x := 2 + 2", executor).isSuccess()); + assertTrue(command.execute("x := 5 + 5", executor).isSuccess()); + + // validate assignment + assertEquals(10, executor.getState().get("x").getResult()); + } + + @Test + public void testAssignmentOfEmptyVar() { + + // z is not defined + StellarResult result = command.execute("x := z", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.isValueNull()); + assertFalse(result.getValue().isPresent()); + + // the value of x is null + assertNull(executor.getState().get("x").getResult()); + } + + @Test + public void testBadAssignmentExpr() { + StellarResult result = command.execute("x := 2 + ", executor); + + // validate the result + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + + // no assignment should have happened + assertFalse(executor.getState().containsKey("x")); + } + + @Test + public void testAssignNull() { + StellarResult result = command.execute("x := NULL", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.isValueNull()); + + // validate assignment + assertNull(executor.getState().get("x").getResult()); + } + + /** + * Assignment with no expression results in an empty string. Is this + * what we would expect? + */ + @Test + public void testNoAssignmentExpr() { + StellarResult result = command.execute("x := ", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // validate assignment + assertEquals("", executor.getState().get("x").getResult()); + } + + @Test + public void testAssignmentWithVar() { + + // define a variable + executor.assign("x", 10, Optional.empty()); + + // execute the assignment expression + StellarResult result = command.execute("y := x + 2", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(12, result.getValue().get()); + + // validate assignment + assertEquals(10, executor.getState().get("x").getResult()); + assertEquals(12, executor.getState().get("y").getResult()); + } + + @Test + public void testAssignmentWithOddWhitespace() { + + StellarResult result = command.execute(" x := 2 + 2", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(4, result.getValue().get()); + + // validate assignment + assertEquals(4, executor.getState().get("x").getResult()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/CommentTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/CommentTest.java new file mode 100644 index 0000000000..0bd5baac12 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/CommentTest.java @@ -0,0 +1,91 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class CommentTest { + + Comment magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new Comment(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("#", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List inputs = Arrays.asList( + "#comment", + " # comment ", + " #comment" + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List inputs = Arrays.asList( + "foo", + " define ", + "bar" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testComment() { + StellarResult result = magic.execute("# this is a comment ", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals("", result.getValue().get()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/DocCommandTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/DocCommandTest.java new file mode 100644 index 0000000000..c3a39c4c26 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/DocCommandTest.java @@ -0,0 +1,86 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.dsl.functions.StringFunctions; +import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; +import org.junit.Before; +import org.junit.Test; + +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertTrue; + +public class DocCommandTest { + + DocCommand command; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the command + command = new DocCommand(); + + // setup a function resolver - only 3 functions have been defined + SimpleFunctionResolver functionResolver = new SimpleFunctionResolver() + .withClass(StringFunctions.ToString.class) + .withClass(StringFunctions.ToLower.class) + .withClass(StringFunctions.ToUpper.class); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(functionResolver, props, Optional.empty()); + executor.init(); + } + + @Test + public void testWithFunction() { + StellarResult result = command.execute("?TO_STRING", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // validate that we have some sort of doc string + assertTrue(result.getValue().toString().length() > 0); + } + + @Test + public void testFunctionNotDefined() { + StellarResult result = command.execute("?INVALID", executor); + + // validate the result + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + } + + @Test + public void testNoFunction() { + StellarResult result = command.execute("?", executor); + + // validate the result + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobalTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobalTest.java new file mode 100644 index 0000000000..4749103302 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobalTest.java @@ -0,0 +1,140 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicDefineGlobalTest { + + MagicDefineGlobal magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicDefineGlobal(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%define", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List inputs = Arrays.asList( + "%define", + " %define ", + "%define x := 2", + " %define x := 2 " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List inputs = Arrays.asList( + "foo", + " define ", + "bar" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testDefine() { + final int expected = 4; + + { + StellarResult result = magic.execute("%define global := 2 + 2", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(expected, result.getValue().get()); + + // ensure global config was updated + assertTrue(executor.getGlobalConfig().containsKey("global")); + assertEquals(expected, executor.getGlobalConfig().get("global")); + } + // + { + // get all globals + StellarResult result = executor.execute("%globals"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + String out = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("{global=4}", out); + } + } + + @Test + public void testNotAssignmentExpression() { + StellarResult result = magic.execute("%define 2 + 2", executor); + + // validate the result + assertTrue(result.isError()); + assertFalse(result.getValue().isPresent()); + assertTrue(result.getException().isPresent()); + + // the global config should not have changed + assertEquals(0, executor.getGlobalConfig().size()); + } + + @Test + public void testMissingExpression() { + StellarResult result = magic.execute("%define", executor); + + // validate the result + assertTrue(result.isError()); + assertFalse(result.getValue().isPresent()); + assertTrue(result.getException().isPresent()); + + // the global config should not have changed + assertEquals(0, executor.getGlobalConfig().size()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctionsTest.java new file mode 100644 index 0000000000..204f9a3677 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctionsTest.java @@ -0,0 +1,136 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.functions.StringFunctions; +import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicListFunctionsTest { + + MagicListFunctions magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicListFunctions(); + + // setup a function resolver - only 3 functions have been defined + SimpleFunctionResolver functionResolver = new SimpleFunctionResolver() + .withClass(StringFunctions.ToString.class) + .withClass(StringFunctions.ToLower.class) + .withClass(StringFunctions.ToUpper.class); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(functionResolver, props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%functions", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List inputs = Arrays.asList( + "%functions", + " %functions ", + "%functions FOO", + " %functions FOO " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List inputs = Arrays.asList( + "foo", + " functions ", + "bar", + "%define" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testFunctions() { + StellarResult result = magic.execute("%functions", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // there are 3 functions that should be returned + String value = ConversionUtils.convert(result.getValue().get(), String.class); + String[] functions = value.split(", "); + assertEquals(3, functions.length); + } + + @Test + public void testFunctionsWithMatch() { + StellarResult result = magic.execute("%functions UPPER", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // only 1 function; TO_UPPER should be returned + String value = ConversionUtils.convert(result.getValue().get(), String.class); + String[] functions = value.split(", "); + assertEquals(1, functions.length); + assertEquals("TO_UPPER", functions[0]); + } + + @Test + public void testFunctionsWithNoMatch() { + StellarResult result = magic.execute("%functions NOMATCH", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // no functions should be returned + String value = ConversionUtils.convert(result.getValue().get(), String.class); + String[] functions = value.trim().split(", "); + assertEquals(1, functions.length); + assertEquals("", functions[0]); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobalsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobalsTest.java new file mode 100644 index 0000000000..f02bfdb36c --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobalsTest.java @@ -0,0 +1,113 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicListGlobalsTest { + + MagicListGlobals magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicListGlobals(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%globals", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List inputs = Arrays.asList( + "%globals", + " %globals ", + "%globals FOO", + " %globals FOO " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List inputs = Arrays.asList( + "foo", + " globals ", + "bar", + "%define" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void test() { + // define some globals + executor.getGlobalConfig().put("x", 2); + + // get all globals + StellarResult result = executor.execute("%globals"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + String out = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("{x=2}", out); + } + + @Test + public void testWithNoGlobals() { + // get all globals + StellarResult result = executor.execute("%globals"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + String out = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("{}", out); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListVariablesTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListVariablesTest.java new file mode 100644 index 0000000000..51ef0ac3df --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListVariablesTest.java @@ -0,0 +1,113 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicListVariablesTest { + + MagicListVariables magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicListVariables(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%vars", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List inputs = Arrays.asList( + "%vars", + " %vars ", + "%vars FOO", + " %vars FOO " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List inputs = Arrays.asList( + "foo", + " vars ", + "bar", + "%define" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void test() { + // define some vars + executor.execute("x := 2 + 2"); + StellarResult result = executor.execute("%vars"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // validate the list of vars + String vars = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("x = 4 via `2 + 2`", vars); + } + + @Test + public void testWithNoVars() { + // there are no vars defined + StellarResult result = executor.execute("%vars"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // validate the list of vars + String vars = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("", vars); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobalTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobalTest.java new file mode 100644 index 0000000000..4db02f4d3b --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobalTest.java @@ -0,0 +1,117 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicUndefineGlobalTest { + + MagicUndefineGlobal magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicUndefineGlobal(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%undefine", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List inputs = Arrays.asList( + "%undefine", + " %undefine ", + "%undefine FOO", + " %undefine FOO " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List inputs = Arrays.asList( + "foo", + " undefine ", + "bar", + "%define" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testUndefine() { + // define a global + executor.getGlobalConfig().put("global", 22); + assertEquals(1, executor.getGlobalConfig().size()); + + // use the magic to undefine the global variable + StellarResult result = magic.execute("%undefine global", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // ensure that the global variable does not exist + assertEquals(0, executor.getGlobalConfig().size()); + } + + @Test + public void testWithNoVariable() { + // define a global + executor.getGlobalConfig().put("global", 22); + assertEquals(1, executor.getGlobalConfig().size()); + + // no arg specifying the var to undefine + StellarResult result = magic.execute("%undefine", executor); + + // validate the result + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + + // ensure that the global variables were not changed + assertEquals(1, executor.getGlobalConfig().size()); + } +} diff --git a/metron-stellar/stellar-zeppelin/README.md b/metron-stellar/stellar-zeppelin/README.md new file mode 100644 index 0000000000..119065842e --- /dev/null +++ b/metron-stellar/stellar-zeppelin/README.md @@ -0,0 +1,160 @@ + + +Stellar Interpreter for Apache Zeppelin +======================================= + +[Apache Zeppelin](https://zeppelin.apache.org/) is a web-based notebook that enables data-driven, interactive data analytics and collaborative documents with SQL, Scala and more. This project provides a means to run the Stellar REPL directly within a Zeppelin Notebook. + +* [Prerequisites](#prerequisites) +* [Installation](#installation) +* [Usage](#usage) + + +Prerequisites +------------- + +* [Apache Zeppelin](https://zeppelin.apache.org/) 0.7.3 + + This is tested with version 0.7.3. Other versions may work, but are not supported. + + +Installation +------------ + +Currently, you need to manually install the Stellar Interpreter in Zeppelin. In the future this step could be automated by the Metron Mpack. + +To install the Stellar Interpreter in your Apache Zeppelin installation, follow these instructions. This is paraphrased from the [Zeppelin docs](https://zeppelin.apache.org/docs/latest/development/writingzeppelininterpreter.html#install-your-interpreter-binary). + +1. Build and install Metron. Metron and its dependencies will be retrieved from your local Maven repository. + + ``` + cd $METRON_HOME + mvn clean install -DskipTests + ``` + +1. If you do not already have Zeppelin installed, [download and unpack Apache Zeppelin](https://zeppelin.apache.org/download.html). The directory in which you unpack Zeppelin will be referred to as `$ZEPPELIN_HOME`. + +1. If Zeppelin was already installed, make sure that it is not running. + +1. Create a settings directory for the Stellar interpreter. + + ``` + mkdir $ZEPPELIN_HOME/interpreter/stellar + cat < $ZEPPELIN_HOME/interpreter/stellar/interpreter-setting.json + [ + { + "group": "stellar", + "name": "stellar", + "className": "org.apache.metron.stellar.zeppelin.StellarInterpreter", + "properties": { + } + } + ] + EOF + ``` + +1. Create a Zeppelin Site file (`$ZEPPELIN_HOME/conf/zeppelin-site.xml`). + + ``` + cp $ZEPPELIN_HOME/conf/zeppelin-site.xml.template $ZEPPELIN_HOME/conf/zeppelin-site.xml + ``` + +1. In the Zeppelin site file, add `org.apache.metron.stellar.zeppelin.StellarInterpreter` to the comma-separated list of Zeppelin interpreters under the `zeppelin.interpreters` property. + + The property will likely look-like the following. + ``` + + zeppelin.interpreters + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.metron.stellar.zeppelin.StellarInterpreter + Comma separated interpreter configurations. First interpreter become a default + + ``` + +1. Start Zeppelin. + + ``` + $ZEPPELIN_HOME/bin/zeppelin-daemon.sh start + ``` + +1. Navigate to Zeppelin running at [http://localhost:8080/](http://localhost:8080/). + +1. Register the Stellar interpreter in Zeppelin. + + 1. Click on the top-right menu item labelled "Anonymous" then choose "Interpreter" in the drop-down that opens. + +1. Configure the Stellar interpreter. + + 1. Click on '**+ Create**' near the top-right. + + 1. Define the following values. + * **Interpreter Name** = `stellar` + * **Interpreter Group** = `stellar` + + 1. Under **Options**, set the following values. + * The interpreter will be instantiated **Per Note** in **isolated** process. + + 1. Under **Dependencies**, define the following fields, then click the "+" icon. Replace the Metron version as required. + * **Artifact** = `org.apache.metron:stellar-zeppelin:0.4.3` + + 1. Click "Save" + +1. Wait for the intrepreter to start. + + 1. Near the title '**stellar**', will be a status icon. This will indicate that it is downloading the dependencies. + + 1. Once the icon is shown as green, the interpreter is ready to work. + +Usage +----- + +1. Create a new notebook. + + 1. Click on "Notebook" > "Create new note". + + 1. Set the default Interpreter to `stellar`. + + When creating the notebook, if you define `stellar` as the default interpreter, then there is no need to enter `%stellar` at the top of each code block. + + If `stellar` is not the default interpreter, then you must enter `%stellar` at the top of a code block containing Stellar code. + +1. In the first block, add the following Stellar, then click Run. + + ``` + 2 in [2,3,4] + ``` + +1. In the next block, check which functions are available to you. + + ``` + %functions + ``` + + You will **only** 'see' the functions defined within `stellar-common` since that is the only library that we added to the interpreter. + +1. To see how additional functions can be added, go back to the Stellar interpreter configuration and add another dependency as follows. + + ``` + org.apache.metron:metron-statistics:0.4.3 + ``` + + Reload the Stellar interpreter and run `%functions` again. You will see the additional functions defined within the `metron-statistics` project. + +1. Auto-completion is also available for Stellar expressions. + + In another block, type 'TO_' then press the CTRL + PERIOD keys. This will trigger the auto-complete mechanism in Stellar and display a list of matching functions or variables. diff --git a/metron-stellar/stellar-zeppelin/pom.xml b/metron-stellar/stellar-zeppelin/pom.xml new file mode 100644 index 0000000000..7809342e11 --- /dev/null +++ b/metron-stellar/stellar-zeppelin/pom.xml @@ -0,0 +1,95 @@ + + + + 4.0.0 + + metron-stellar + org.apache.metron + 0.4.3 + + stellar-zeppelin + Stellar Interpreter for Apache Zeppelin + stellar-zeppelin + + + org.apache.zeppelin + zeppelin-interpreter + 0.7.3 + + + org.apache.metron + stellar-common + ${project.parent.version} + + + org.mockito + mockito-all + ${global_mockito_version} + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${global_jar_version} + + + + test-jar + + + + + + org.antlr + antlr4-maven-plugin + ${global_antlr_version} + + ${basedir}/src/main/java + + + + + antlr4 + + + + + + maven-assembly-plugin + + src/main/assembly/assembly.xml + + + + make-assembly + package + + single + + + + + + + + src/main/resources + + + + diff --git a/metron-stellar/stellar-zeppelin/src/main/assembly/assembly.xml b/metron-stellar/stellar-zeppelin/src/main/assembly/assembly.xml new file mode 100644 index 0000000000..090835b872 --- /dev/null +++ b/metron-stellar/stellar-zeppelin/src/main/assembly/assembly.xml @@ -0,0 +1,30 @@ + + + + archive + + tar.gz + + false + + + ${project.basedir}/target + + ${project.artifactId}-${project.version}.jar + + lib + true + + + \ No newline at end of file diff --git a/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java b/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java new file mode 100644 index 0000000000..58287dce09 --- /dev/null +++ b/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.zeppelin; + +import static org.apache.zeppelin.interpreter.InterpreterResult.Code.ERROR; +import static org.apache.zeppelin.interpreter.InterpreterResult.Code.SUCCESS; +import static org.apache.zeppelin.interpreter.InterpreterResult.Type.TEXT; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.metron.stellar.common.shell.DefaultStellarAutoCompleter; +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarAutoCompleter; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Zeppelin Interpreter for Stellar. + */ +public class StellarInterpreter extends Interpreter { + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Executes the Stellar expressions. + * + *

Zeppelin will handle isolation and how the same executor is or is not used across + * multiple notebooks. This is configurable by the user. + * + *

See https://zeppelin.apache.org/docs/latest/manual/interpreters.html#interpreter-binding-mode. + */ + private StellarShellExecutor executor; + + /** + * Handles auto-completion for Stellar expressions. + */ + private StellarAutoCompleter autoCompleter; + + public StellarInterpreter(Properties properties) { + super(properties); + this.autoCompleter = new DefaultStellarAutoCompleter(); + } + + @Override + public void open() { + try { + executor = createExecutor(); + + } catch (Exception e) { + LOG.error("Unable to create a StellarShellExecutor", e); + } + } + + @Override + public void close() { + // nothing to do + } + + @Override + public InterpreterResult interpret(final String input, InterpreterContext context) { + InterpreterResult result; + try { + + // execute the input + StellarResult stellarResult = executor.execute(input); + if(stellarResult.isSuccess()) { + + // on success - if no result, use a blank value + Object value = stellarResult.getValue().orElse(""); + String text = value.toString(); + result = new InterpreterResult(SUCCESS, TEXT, text); + + } else if(stellarResult.isError()) { + + // an error occurred + Optional e = stellarResult.getException(); + String message = getErrorMessage(e, input); + result = new InterpreterResult(ERROR, TEXT, message); + + } else { + + // should never happen + throw new IllegalStateException("Unexpected error. result=" + stellarResult); + } + + } catch(Throwable t) { + + // unexpected exception + String message = getErrorMessage(Optional.of(t), input); + result = new InterpreterResult(ERROR, TEXT, message); + } + + return result; + } + + @Override + public void cancel(InterpreterContext context) { + // there is no way to cancel the execution of a Stellar expression + } + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + // unable to provide progress + return 0; + } + + @Override + public List completion(String buf, int cursor) { + + // use the autoCompleter to return a list of completes to Zeppelin + List completes = new ArrayList<>(); + for(String candidate : autoCompleter.autoComplete(buf)) { + completes.add(new InterpreterCompletion(candidate, candidate)); + } + + return completes; + } + + /** + * Generates an error message that is shown to the user. + * + * @param e An optional exception that occurred. + * @param input The user input that led to the error condition. + * @return An error message for the user. + */ + private String getErrorMessage(Optional e, String input) { + String message; + if(e.isPresent()) { + + // base the error message on the exception + String error = ExceptionUtils.getRootCauseMessage(e.get()); + String trace = ExceptionUtils.getStackTrace(e.get()); + message = error + System.lineSeparator() + trace; + + } else { + // no exception provided; create generic error message + message = "Invalid expression: " + input; + } + + return message; + } + + /** + * Create an executor that will run the Stellar code for the Zeppelin Notebook. + * @return The stellar executor. + */ + private StellarShellExecutor createExecutor() throws Exception { + + Properties props = getProperty(); + StellarShellExecutor executor = new DefaultStellarShellExecutor(props, Optional.empty()); + + // register the auto-completer to be notified + executor.addSpecialListener((magic) -> autoCompleter.addCandidateFunction(magic.getCommand())); + executor.addFunctionListener((fn) -> autoCompleter.addCandidateFunction(fn.getName())); + executor.addVariableListener((name, val) -> autoCompleter.addCandidateVariable(name)); + + executor.init(); + return executor; + } + + /** + * Returns the executor used to execute Stellar expressions. + * @return The executor of Stellar expressions. + */ + public StellarShellExecutor getExecutor() { + return executor; + } +} diff --git a/metron-stellar/stellar-zeppelin/src/main/resources/interpreter-setting.json b/metron-stellar/stellar-zeppelin/src/main/resources/interpreter-setting.json new file mode 100644 index 0000000000..cbd974acd0 --- /dev/null +++ b/metron-stellar/stellar-zeppelin/src/main/resources/interpreter-setting.json @@ -0,0 +1,12 @@ +[ + { + "group": "stellar", + "name": "stellar", + "className": "org.apache.metron.stellar.zeppelin.StellarInterpreter", + "properties": { + }, + "editor": { + "completionKey": "TAB" + } + } +] diff --git a/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java new file mode 100644 index 0000000000..363938e699 --- /dev/null +++ b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.zeppelin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import com.google.common.collect.Iterables; +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Properties; + +/** + * Tests the StellarInterpreter. + */ +public class StellarInterpreterTest { + + private StellarInterpreter interpreter; + private InterpreterContext context; + + @Before + public void setup() { + Properties props = new Properties(); + interpreter = new StellarInterpreter(props); + interpreter.open(); + + context = mock(InterpreterContext.class); + } + + /** + * Ensure that we can run Stellar code in the interpreter. + */ + @Test + public void testExecuteStellar() { + InterpreterResult result = interpreter.interpret("2 + 2", context); + + // validate the result + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertEquals("4", message.getData()); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * Ensure that Stellar lists are displayed correctly in Zeppelin. + */ + @Test + public void testExecuteWithStellarList() { + final String expected = "[1, 2, 3, 4, 5]"; + InterpreterResult result = interpreter.interpret("[1,2,3,4,5]", context); + + // validate the result + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertEquals(expected, message.getData()); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * Ensure that Stellar maps are displayed correctly in Zeppelin. + */ + @Test + public void testExecuteWithStellarMap() { + final String expected = "{foo=2, key=val}"; + InterpreterResult result = interpreter.interpret("{ 'foo':2, 'key':'val' }", context); + + // validate the result + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertEquals(expected, message.getData()); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * Ensure that 'bad' Stellar code is handled correctly by the interpreter. + */ + @Test + public void testExecuteBadStellar() { + InterpreterResult result = interpreter.interpret("2 + ", context); + + // validate the result + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertTrue(message.getData().length() > 0); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * The interpreter should handle if no value is returned. Some Stellar expressions + * will result in no value. + */ + @Test + public void testExecuteNoop() { + + // x is undefined and will have no result + InterpreterResult result = interpreter.interpret("x", context); + + // validate the result + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertEquals(0, message.getData().length()); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * The interpreter should support auto-completion. + */ + @Test + public void testAutoCompletion() { + + // the user's input that needs auto-completed + final String buffer = "TO_"; + + // the cursor is at the end of the buffer + int cursor = buffer.length(); + + List completions = interpreter.completion(buffer, cursor); + + // expect some completions to be offered + assertTrue(completions.size() > 0); + + for(InterpreterCompletion iCompletion: completions) { + String completion = iCompletion.getValue(); + + // the auto-complete should include an open paren + assertEquals("(", completion.substring(completion.length() - 1)); + + // the candidate should be a valid, defined function + String function = completion.substring(0, completion.length() - 1); + Iterable allFunctions = interpreter.getExecutor().getFunctionResolver().getFunctions(); + String definedFunction = Iterables.find(allFunctions, (fn) -> StringUtils.equals(fn, function)); + assertEquals(function, definedFunction); + } + } + + /** + * What happens when we have nothing useful to auto-complete? + */ + @Test + public void testAutoCompletionWithNoCompletions() { + + // the user's input that needs auto-completed + final String buffer = "NOTHING_AUTOCOMPLETES_THIS_"; + + // the cursor is at the end of the buffer + int cursor = buffer.length(); + + // perform auto-completion + List completions = interpreter.completion(buffer, cursor); + + // expect no completions + assertEquals(0, completions.size()); + } +}