diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 1e73eb1b9a..aabf9836ec 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -177,6 +177,8 @@ commons-lang:commons-lang:jar:2.4:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.5:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.6:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.6:provided,ASLv2,http://commons.apache.org/lang/ +commons-lang:commons-lang:jar:3.7:compile,ASLv2,http://commons.apache.org/lang/ +commons-lang:commons-lang:jar:3.7:provided,ASLv2,http://commons.apache.org/lang/ commons-logging:commons-logging:jar:1.1.1:compile,ASLv2,http://commons.apache.org/logging commons-logging:commons-logging:jar:1.1.3:compile,ASLv2,http://commons.apache.org/proper/commons-logging/ commons-logging:commons-logging:jar:1.2:compile,ASLv2,http://commons.apache.org/proper/commons-logging/ diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java index df80691ddd..e89341dd50 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java @@ -20,6 +20,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableList; +import org.apache.metron.common.field.transformation.StellarTransformation; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.common.field.transformation.FieldTransformation; import org.apache.metron.common.field.transformation.FieldTransformations; @@ -28,11 +31,14 @@ import java.io.Serializable; import java.util.*; +@StellarConfigurationList(name = "FieldTransformer") public class FieldTransformer implements Serializable { private List input = new ArrayList<>(); private List output; private FieldTransformation transformation; private String transformationName; + @StellarExpressionMap(name = "Field Mapping", + qualifyWithField = "transformation", qualifyWithFieldType = StellarTransformation.class) private LinkedHashMap config = new LinkedHashMap<>(); private boolean initialized = false; public FieldTransformer() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java index 2d0ccd8027..af4bfd7a51 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java @@ -26,7 +26,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; +@StellarConfiguration public class SensorParserConfig implements Serializable { private String parserClassName; @@ -229,6 +232,7 @@ public void setWriterClassName(String classNames) { this.writerClassName = classNames; } private Map parserConfig = new HashMap<>(); + @StellarConfigurationList(name = "fieldTransformations") private List fieldTransformations = new ArrayList<>(); public List getFieldTransformations() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ZookeeperConfigurationProvider.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ZookeeperConfigurationProvider.java new file mode 100644 index 0000000000..b76ae934f9 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ZookeeperConfigurationProvider.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.metron.common.configuration; + +import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; +import static org.apache.metron.common.configuration.ConfigurationType.PARSER; +import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; + +import java.lang.invoke.MethodHandles; +import java.util.LinkedList; +import java.util.List; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.configuration.profiler.ProfileConfig; +import org.apache.metron.common.configuration.profiler.ProfilerConfig; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.stellar.common.utils.validation.ExpressionConfigurationHolder; +import org.apache.metron.stellar.common.utils.validation.StellarZookeeperConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer; +import org.apache.zookeeper.KeeperException.NoNodeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@code ZookeeperConfigurationProvider} is used to report all of the configured / deployed Stellar statements in + * the system. + */ +public class ZookeeperConfigurationProvider implements StellarZookeeperConfigurationProvider { + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + /** + * Default constructor. + */ + public ZookeeperConfigurationProvider() { + } + + @Override + public String getName() { + return "Apache Metron"; + } + + @Override + public List provideContainers(CuratorFramework client, + ErrorConsumer errorConsumer) { + List holders = new LinkedList<>(); + visitParserConfigs(client, holders, errorConsumer); + visitEnrichmentConfigs(client, holders, errorConsumer); + visitProfilerConfigs(client, holders, errorConsumer); + return holders; + } + + private void visitParserConfigs(CuratorFramework client, + List holders, ErrorConsumer errorConsumer) { + List children = null; + + try { + children = client.getChildren().forPath(PARSER.getZookeeperRoot()); + } catch (Exception e) { + LOG.error("Exception getting parser configurations", e); + return; + } + for (String child : children) { + try { + byte[] data = client.getData().forPath(PARSER.getZookeeperRoot() + "/" + child); + SensorParserConfig parserConfig = SensorParserConfig.fromBytes(data); + ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder( + String.format("%s/%s", getName(), PARSER.toString()), parserConfig.getSensorTopic(), + parserConfig); + holders.add(holder); + } catch (Exception e) { + errorConsumer.consume(String.format("%s/%s/%s", getName(), PARSER.toString(), child), e); + } + } + } + + @SuppressWarnings("unchecked") + private void visitEnrichmentConfigs(CuratorFramework client, + List holders, ErrorConsumer errorConsumer) { + List children = null; + + try { + children = client.getChildren().forPath(ENRICHMENT.getZookeeperRoot()); + } catch (Exception e) { + LOG.error("Exception getting enrichment configurations", e); + return; + } + + for (String child : children) { + + try { + byte[] data = client.getData().forPath(ENRICHMENT.getZookeeperRoot() + "/" + child); + // Certain parts of the SensorEnrichmentConfig do Stellar Verification on their + // own as part of deserialization, where the bean spec will call the setter, which has + // been wired with stellar verification calls. There is no avoiding this. + // + // In cases where those parts of the config are in fact the parts that have invalid + // Stellar statements, we will fail during the JSON load before we get to ANY config + // contained in the SensorEnrichmentConfig. + // + // I have left the code to properly check all the configuration parts for completeness + // on the reporting side ( the report initiator may want to list successful evals), even + // though they can be executed, then they will never fail. + final SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig + .fromBytes(data); + ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder( + String.format("%s/%s", getName(), ENRICHMENT.toString()), child, + sensorEnrichmentConfig); + holders.add(holder); + } catch (Exception e) { + errorConsumer + .consume(String.format("%s/%s/%s", getName(), ENRICHMENT.toString(), child), e); + } + } + } + + private void visitProfilerConfigs(CuratorFramework client, + List holders, ErrorConsumer errorConsumer) { + try { + byte[] profilerConfigData = null; + try { + profilerConfigData = client.getData().forPath(PROFILER.getZookeeperRoot()); + } catch (NoNodeException e) { + LOG.error("Exception getting profiler configurations", e); + return; + } + + ProfilerConfig profilerConfig = JSONUtils.INSTANCE + .load(new String(profilerConfigData), ProfilerConfig.class); + profilerConfig.getProfiles().forEach((ProfileConfig pc) -> { + ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder( + String.format("%s/%s", getName(), PROFILER.toString()), pc.getProfile(), pc); + holders.add(holder); + }); + } catch (Exception e) { + errorConsumer.consume(String.format("%s/%s", getName(), PROFILER.toString()), e); + } + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java index 3261c9f19f..07fda8564f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java @@ -24,8 +24,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; +@StellarConfiguration public class EnrichmentConfig { + @StellarExpressionMap(innerMapKeys = {"stellar","config"}) private Map fieldMap = new HashMap<>(); private Map enrichmentConfigs = new HashMap<>(); private Map> fieldToTypeMap = new HashMap<>(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java index 2b4f5a89c5..3f94ed4d11 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java @@ -24,10 +24,14 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +@StellarConfiguration public class SensorEnrichmentConfig { + @StellarConfiguration private EnrichmentConfig enrichment = new EnrichmentConfig(); + @StellarConfiguration private ThreatIntelConfig threatIntel = new ThreatIntelConfig(); private Map configuration = new HashMap<>(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java index 94ab0c8d63..d9f93ae644 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java @@ -17,6 +17,9 @@ */ package org.apache.metron.common.configuration.enrichment.threatintel; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; + /** * This class represents a rule that is used to triage threats. * @@ -29,6 +32,7 @@ * Tuning the threat triage process involves creating one or more rules, adjusting * the score of each rule, and changing the way that each rule's score is aggregated. */ +@StellarConfiguration public class RiskLevelRule { /** @@ -45,6 +49,7 @@ public class RiskLevelRule { * A predicate, in the form of a Stellar expression, that determines whether * the rule is applied to an alert or not. This field is required. */ + @StellarExpressionField(name = "rule") String rule; /** @@ -60,6 +65,7 @@ public class RiskLevelRule { * This is expected to be a valid Stellar expression and can refer to any of the * fields within the message itself. */ + @StellarExpressionField(name = "reason") String reason; public String getName() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java index 3b34a0af37..a3721e0955 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java @@ -19,8 +19,11 @@ package org.apache.metron.common.configuration.enrichment.threatintel; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +@StellarConfiguration public class ThreatIntelConfig extends EnrichmentConfig { + @StellarConfiguration private ThreatTriageConfig triageConfig = new ThreatTriageConfig(); public ThreatTriageConfig getTriageConfig() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java index 961f9d56ea..81d386559d 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java @@ -29,9 +29,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; public class ThreatTriageConfig { + @StellarConfigurationList(name = "riskLevelRules") private List riskLevelRules = new ArrayList<>(); private Aggregators aggregator = Aggregators.MAX; private Map aggregationConfig = new HashMap<>(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java index 6205fbf377..c66892cadd 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java @@ -26,10 +26,15 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; /** * The definition of a single Profile. */ +@StellarConfiguration public class ProfileConfig implements Serializable { /** @@ -52,6 +57,7 @@ public class ProfileConfig implements Serializable { * is only applied to a profile if this condition is true. This allows a profile * to filter the messages that it receives. */ + @StellarExpressionField(name = "onlyif") private String onlyif = "true"; /** @@ -61,6 +67,7 @@ public class ProfileConfig implements Serializable { * period the expression is executed once and stored in a variable with the given * name. */ + @StellarExpressionMap(name = "init") private Map init = new HashMap<>(); /** @@ -68,12 +75,14 @@ public class ProfileConfig implements Serializable { * A map is expected where the key is the variable name and the value is a Stellar * expression. The map can include 0 or more variables/expressions. */ + @StellarExpressionMap(name = "update") private Map update = new HashMap<>(); /** * A list of Stellar expressions that is executed in order and used to group the * resulting profile data. */ + @StellarExpressionList(name = "groupBy") private List groupBy = new ArrayList<>(); /** @@ -81,6 +90,7 @@ public class ProfileConfig implements Serializable { * expression(s) are expected to in some way summarize the messages that were applied * to the profile over the window period. */ + @StellarConfiguration private ProfileResult result; /** diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java index 55642a938a..f12fb1c3da 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java @@ -19,10 +19,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; /** * Defines the 'result' field of a Profile definition. */ +@StellarConfiguration public class ProfileResult { /** @@ -30,6 +32,7 @@ public class ProfileResult { * a measurement that is persisted in the profile store. */ @JsonProperty("profile") + @StellarConfiguration private ProfileResultExpressions profileExpressions; /** @@ -38,6 +41,7 @@ public class ProfileResult { * triage. */ @JsonProperty("triage") + @StellarConfiguration private ProfileTriageExpressions triageExpressions; @JsonCreator diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java index 82af223d86..57ba7a1275 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java @@ -19,14 +19,18 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; /** * A Stellar expression that is executed to produce a single * measurement that is persisted within the profile store. */ +@StellarConfiguration public class ProfileResultExpressions { @JsonIgnore + @StellarExpressionField(name = "expression") private String expression; @JsonCreator diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java index fbe1706b4d..a1a57d26f8 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java @@ -22,6 +22,8 @@ import java.util.HashMap; import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; /** * A set of Stellar expressions that are executed to produce a single @@ -30,6 +32,7 @@ * The result of evaluating each expression are made available, keyed * by the given name, to the threat triage process. */ +@StellarConfiguration public class ProfileTriageExpressions { /** @@ -40,6 +43,7 @@ public class ProfileTriageExpressions { * or map of basic data types that can be serialized. */ @JsonIgnore + @StellarExpressionMap(name = "expressions") private Map expressions; @JsonCreator diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java index 0bdb7e2ffc..0637fb76af 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java @@ -20,16 +20,20 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; import java.util.Optional; /** * The configuration object for the Profiler, which may contain many Profile definitions. */ +@StellarConfiguration public class ProfilerConfig implements Serializable { /** * One or more profile definitions. */ + @StellarConfigurationList(name = "profiles") private List profiles = new ArrayList<>(); /** diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java index 2a22e2168b..bbdacec1ad 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java @@ -18,13 +18,16 @@ package org.apache.metron.common.field.transformation; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; -import org.apache.metron.stellar.dsl.VariableResolver; -import org.apache.metron.stellar.common.StellarProcessor; - -import java.util.*; public class StellarTransformation implements FieldTransformation { @Override diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ZookeeperConfigurationProviderTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ZookeeperConfigurationProviderTest.java new file mode 100644 index 0000000000..f1d4fa502e --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ZookeeperConfigurationProviderTest.java @@ -0,0 +1,308 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.configuration; + +import java.util.LinkedList; +import java.util.List; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.test.TestingServer; +import org.apache.metron.stellar.common.utils.validation.ExpressionConfigurationHolder; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class ZookeeperConfigurationProviderTest { + + private TestingServer testZkServer; + private String zookeeperUrl; + private CuratorFramework client; + private List expected; + private List expectedStatements; + private List foundStatements; + @Before + public void setUp() throws Exception { + testZkServer = new TestingServer(true); + zookeeperUrl = testZkServer.getConnectString(); + client = ConfigurationsUtils.getClient(zookeeperUrl); + expected = new LinkedList<>(); + expected.add("Apache Metron/PARSER/squid"); + expected.add("Apache Metron/ENRICHMENT/snort"); + expected.add("Apache Metron/PROFILER/example2"); + expected.add("Apache Metron/PROFILER/example1"); + expected.add("Apache Metron/PROFILER/percentiles"); + + expectedStatements = new LinkedList<>(); + expectedStatements.add("Apache Metron/PARSER/squid/fieldTransformations/0/Field Mapping/full_hostname"); + expectedStatements.add("Apache Metron/PARSER/squid/fieldTransformations/0/Field Mapping/domain_without_subdomains"); + expectedStatements.add("Apache Metron/ENRICHMENT/snort/enrichment/default/stellar/config/foo"); + expectedStatements.add("Apache Metron/ENRICHMENT/snort/enrichment/default/stellar/config/ALL_CAPS"); + expectedStatements.add("Apache Metron/ENRICHMENT/snort/threatIntel/triageConfig/riskLevelRules/0/rule"); + expectedStatements.add("Apache Metron/PROFILER/example2/onlyif"); + expectedStatements.add("Apache Metron/PROFILER/example2/init/num_dns"); + expectedStatements.add("Apache Metron/PROFILER/example2/init/num_http"); + expectedStatements.add("Apache Metron/PROFILER/example2/update/num_dns"); + expectedStatements.add("Apache Metron/PROFILER/example2/update/num_http"); + expectedStatements.add("Apache Metron/PROFILER/example2/result/profileExpressions/expression"); + expectedStatements.add("Apache Metron/PROFILER/example1/onlyif"); + expectedStatements.add("Apache Metron/PROFILER/example1/init/total_bytes"); + expectedStatements.add("Apache Metron/PROFILER/example1/update/total_bytes"); + expectedStatements.add("Apache Metron/PROFILER/example1/result/profileExpressions/expression"); + expectedStatements.add("Apache Metron/PROFILER/percentiles/onlyif"); + expectedStatements.add("Apache Metron/PROFILER/percentiles/init/s"); + expectedStatements.add("Apache Metron/PROFILER/percentiles/update/s"); + expectedStatements.add("Apache Metron/PROFILER/percentiles/result/profileExpressions/expression"); + + foundStatements = new LinkedList<>(); + + client.start(); + } + + @After + public void tearDown() throws Exception { + client.close(); + testZkServer.close(); + testZkServer.stop(); + } + + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "squid", + * "parserConfig": { + * "grokPath": "/patterns/squid", + * "patternLabel": "SQUID_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR", + * "output" : [ "full_hostname", "domain_without_subdomains" ], + * "config" : { + * "full_hostname" : "URL_TO_HOST(url)", + * "domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + * } + * } + * ] + * } + */ + @Multiline + public static String squidParserConfig; + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "squid", + * "parserConfig": { + * "grokPath": "/patterns/squid", + * "patternLabel": "SQUID_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR", + * "output" : [ "full_hostname", "domain_without_subdomains" ], + * "config" : { + * "full_hostname" : "URL_TO_HOST(url)", + * "domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + * } + * } + */ + @Multiline + public static String badParserConfig; + + + /** + * { + * "enrichment" : { + * "fieldMap": + * { + * "geo": ["ip_dst_addr", "ip_src_addr"], + * "host": ["host"], + * "stellar" : { + * "type" : "STELLAR", + * "config" : { + * "foo" : "1 + 1", + * "ALL_CAPS" : "TO_UPPER(source.type)" + * } + * } + * } + * }, + * "threatIntel" : { + * "fieldMap": + * { + * "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] + * }, + * "fieldToTypeMap": + * { + * "ip_src_addr" : ["malicious_ip"], + * "ip_dst_addr" : ["malicious_ip"] + * }, + * "triageConfig" : { + * "riskLevelRules" : [ + * { + * "rule" : "not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))", + * "score" : 10 + * } + * ], + * "aggregator" : "MAX" + * } + * } + * } + */ + @Multiline + public static String snortEnrichmentConfig; + + /** + * { + * "enrichment" : { + * "fieldMap": + * { + * "geo": ["ip_dst_addr", "ip_src_addr"], + * "host": ["host"] + * } + * }, + * "threatIntel" : { + * "fieldMap": + * { + * "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] + * }, + * "fieldToTypeMap": + * { + * "ip_src_addr" : ["malicious_ip"], + * "ip_dst_addr" : ["malicious_ip"] + * }, + * "triageConfig" : { + * "riskLevelRules" : [ + * { + * "rule" : "not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24')))", + * "score" : 10 + * } + * ], + * "aggregator" : "MAX" + * } + * } + * } + */ + @Multiline + public static String badEnrichmentConfig; + + /** + * { + * "profiles": [ + * { + * "profile": "example2", + * "foreach": "ip_src_addr", + * "onlyif": "protocol == 'DNS' or protocol == 'HTTP'", + * "init": { + * "num_dns": 1.0, + * "num_http": 1.0 + * }, + * "update": { + * "num_dns": "num_dns + (if protocol == 'DNS' then 1 else 0)", + * "num_http": "num_http + (if protocol == 'HTTP' then 1 else 0)" + * }, + * "result": "num_dns / num_http" + * }, + * { + * "profile": "example1", + * "foreach": "ip_src_addr", + * "onlyif": "protocol == 'HTTP'", + * "init": { + * "total_bytes": 0.0 + * }, + * "update": { + * "total_bytes": "total_bytes + bytes_in" + * }, + * "result": "total_bytes", + * "expires": 30 + * }, + * { + * "profile": "percentiles", + * "foreach": "ip_src_addr", + * "onlyif": "protocol == 'HTTP'", + * "init": { "s": "STATS_INIT(100)" }, + * "update": { "s": "STATS_ADD(s, length)" }, + * "result": "STATS_PERCENTILE(s, 0.7)" + * } + * ] + * } + */ + @Multiline + public static String profilerConfig; + + @Test + public void testProvideAllConfiguredNodes() throws Exception { + ConfigurationsUtils + .writeSensorParserConfigToZookeeper("squid", squidParserConfig.getBytes(), zookeeperUrl); + + ConfigurationsUtils + .writeSensorEnrichmentConfigToZookeeper("snort", snortEnrichmentConfig.getBytes(), + zookeeperUrl); + ConfigurationsUtils.writeProfilerConfigToZookeeper(profilerConfig.getBytes(), client); + + ZookeeperConfigurationProvider zookeeperConfigurationProvider = new ZookeeperConfigurationProvider(); + List holders = zookeeperConfigurationProvider + .provideContainers(client, + (names, error) -> Assert.assertTrue("Should not get errors", false)); + holders.forEach((h) -> { + try { + h.discover(); + h.visit((p,s) -> { + foundStatements.add(p); + },(s,e) -> { + Assert.assertTrue(e.getMessage(),false); + }); + } catch (Exception e) { + Assert.assertTrue(e.getMessage(), false); + } + }); + + holders.forEach((h) -> expected.remove((h.getFullName()))); + Assert.assertTrue(expected.isEmpty()); + + foundStatements.removeAll(expectedStatements); + Assert.assertTrue(foundStatements.isEmpty()); + } + + @Test + public void testErrorCalledWithBadParserConfig() throws Exception { + // calling this way to avoid getting the serialize error in ConfigurationUtils instead of visit + ConfigurationsUtils + .writeToZookeeper(ConfigurationType.PARSER.getZookeeperRoot() + "/" + "squid", + badParserConfig.getBytes(), client); + ZookeeperConfigurationProvider zookeeperConfigurationProvider = new ZookeeperConfigurationProvider(); + zookeeperConfigurationProvider.provideContainers(client, (names, error) -> { + Assert.assertTrue("Should get errors", true); + }); + } + + @Test(expected = RuntimeException.class) + public void testExceptionThrownGetsBadThreatConfig() throws Exception { + ConfigurationsUtils + .writeSensorEnrichmentConfigToZookeeper("snort", badEnrichmentConfig.getBytes(), + zookeeperUrl); + ZookeeperConfigurationProvider zookeeperConfigurationProvider = new ZookeeperConfigurationProvider(); + zookeeperConfigurationProvider.provideContainers(client, + (names, error) -> Assert.assertTrue("Should not get errors", false)); + } +} \ No newline at end of file diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md index bf939c29ed..09f1ea5c85 100644 --- a/metron-platform/metron-management/README.md +++ b/metron-platform/metron-management/README.md @@ -40,6 +40,7 @@ project. * [Indexing Functions](#indexing-functions) * [Enrichment Functions](#enrichment-functions) * [Threat Triage Functions](#threat-triage-functions) + * [Validation Functions](#validation-functions) * [Examples](#examples) * [Iterate to Find a Valid Grok Pattern](#iterate-to-find-a-valid-grok-pattern) * [Manage Stellar Field Transformations](#manage-stellar-field-transformations) @@ -57,6 +58,7 @@ The functions are split roughly into a few sections: * Parser functions - Functions surrounding adding, viewing, and removing Parser functions. * Enrichment functions - Functions surrounding adding, viewing and removing Stellar enrichments as well as managing batch size, batch timeout, and index names for the enrichment topology configuration * Threat Triage functions - Functions surrounding adding, viewing and removing threat triage functions. +* Validation functions - Functions to validate stellar rules contained in configurations. ### Grok Functions @@ -263,6 +265,13 @@ The functions are split roughly into a few sections: * aggregatorConfig - Optional config for aggregator * Returns: The String representation of the enrichment config +### Validation Functions +* `VALIDATE_STELLAR_RULE_CONFIGS` + * Description: Attempts to validate deployed Stellar expressions by ensuring they compile. This is useful when expressions may have been valid when deployed, but may have been invalidated by a language change after that + * Input: + * wrap - Optional. The Length of the string to wrap the columns + * Returns: A table of validation results + ## Deployment Instructions * Clusters installed via Ambari Management Pack (default) * Automatically deployed diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java new file mode 100644 index 0000000000..7d1ddbd34f --- /dev/null +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.metron.management; + +import com.jakewharton.fliptables.FlipTable; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.apache.commons.lang3.text.WordUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.validation.StellarValidator; +import org.apache.metron.stellar.common.utils.validation.validators.StellarSimpleValidator; +import org.apache.metron.stellar.common.utils.validation.validators.StellarZookeeperBasedValidator; +import org.apache.metron.stellar.common.utils.validation.ValidationResult; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; + +/** + * The VALIDATE namespace is used for functions that validate Stellar rule strings. + */ +public class ValidationFunctions { + + private static final String[] headers = new String[]{"PATH", "RULE", "ERROR", "VALID"}; + + @Stellar(namespace = "VALIDATE", name = "STELLAR_RULE_CONFIGS", description = + "Attempts to validate deployed Stellar expressions by ensuring they compile." + + "This is useful when expressions may have been valid when deployed, but may have been" + + "invalidated by a language change after that", params = { + "wrap : Optional. The Length of string to wrap the columns"}, returns = "A table of validation results") + public static class DiscoverAndValidateStellarRules implements StellarFunction { + + @Override + public Object apply(List args, Context context) { + + int wordWrap = -1; + if (args.size() > 0) { + wordWrap = ConversionUtils.convert(args.get(0), Integer.class); + } + + final int wrapSize = wordWrap; + Optional clientOpt = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + if (!clientOpt.isPresent()) { + throw new IllegalStateException( + "VALIDATE_STELLAR_RULE_CONFIGS requires zookeeper. Please connect to zookeeper."); + } + CuratorFramework client = (CuratorFramework) clientOpt.get(); + final ArrayList dataList = new ArrayList<>(); + + // validate the zookeeper configurations + // when we support other validators, we will call them as well + validate(new StellarZookeeperBasedValidator(client), dataList, wrapSize); + validate(new StellarSimpleValidator(), dataList, wrapSize); + + if (dataList.isEmpty()) { + return FlipTable.of(headers, new String[][]{}); + } + + String[][] data = new String[dataList.size()][headers.length]; + for (int i = 0; i < dataList.size(); ++i) { + data[i] = dataList.get(i); + } + return FlipTable.of(headers, data); + } + + private static String toWrappedString(Object o, int wrap) { + if (o == null) { + return ""; + } + String s = "" + o; + if (wrap <= 0) { + return s; + } + return WordUtils.wrap(s, wrap); + } + + private void validate(StellarValidator validator, ArrayList dataList, int wrapSize) { + Iterable result = validator.validate(); + + result.forEach((v) -> { + dataList.add(new String[]{toWrappedString(v.getRulePath(), wrapSize), + toWrappedString(v.getRule(), wrapSize), toWrappedString(v.getError(), wrapSize), + toWrappedString(String.valueOf(v.isValid()), wrapSize)}); + }); + + validator = new StellarSimpleValidator(); + result = validator.validate(); + + if (dataList.isEmpty()) { + return; + } + + String[][] data = new String[dataList.size()][headers.length]; + for (int i = 0; i < dataList.size(); ++i) { + data[i] = dataList.get(i); + } + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } + +} diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ValidationFunctionsIntegrationTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ValidationFunctionsIntegrationTest.java new file mode 100644 index 0000000000..534bcb15c0 --- /dev/null +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ValidationFunctionsIntegrationTest.java @@ -0,0 +1,180 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.management; + +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.dsl.Context.Capabilities.ZOOKEEPER_CLIENT; + +import java.util.HashMap; +import java.util.Properties; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.integration.BaseIntegrationTest; +import org.apache.metron.integration.ComponentRunner; +import org.apache.metron.integration.components.ZKServerComponent; +import org.apache.metron.stellar.dsl.Context; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ValidationFunctionsIntegrationTest extends BaseIntegrationTest { + + private static ZKServerComponent zkServerComponent; + private static ComponentRunner runner; + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "foo", + * "parserConfig": { + * "grokPath": "/patterns/foo", + * "patternLabel": "FOO_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR" + * ,"output" : [ "full_hostname", "domain_without_subdomains" ] + * ,"config" : { + * "full_hostname" : "URL_TO_HOST(url)" + * ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + * } + * } + * ] + * } + */ + @Multiline + private static String config; + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "bar", + * "parserConfig": { + * "grokPath": "/patterns/bar", + * "patternLabel": "BAR_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR" + * ,"output" : [ "full_hostname", "domain_without_subdomains" ] + * ,"config" : { + * "full_hostname" : "URL_TO_HOST(url)=" + * ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)=" + * } + * } + * ] + * } + */ + @Multiline + private static String badConfig; + + /** + *╔═════════════════════════════════════════════════════════════════════════════════════════╤══════════════════════════════════════════╤═════════════════════════════════════════════════════╤═══════╗ + *║ PATH │ RULE │ ERROR │ VALID ║ + *╠═════════════════════════════════════════════════════════════════════════════════════════╪══════════════════════════════════════════╪═════════════════════════════════════════════════════╪═══════╣ + *║ Apache Metron/PARSER/bar/fieldTransformations/0/Field Mapping/full_hostname │ URL_TO_HOST(url)= │ Syntax error @ 1:16 token recognition error at: '=' │ false ║ + *╟─────────────────────────────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┼─────────────────────────────────────────────────────┼───────╢ + *║ Apache Metron/PARSER/bar/fieldTransformations/0/Field Mapping/domain_without_subdomains │ DOMAIN_REMOVE_SUBDOMAINS(full_hostname)= │ Syntax error @ 1:39 token recognition error at: '=' │ false ║ + *╟─────────────────────────────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┼─────────────────────────────────────────────────────┼───────╢ + *║ Apache Metron/PARSER/foo/fieldTransformations/0/Field Mapping/full_hostname │ URL_TO_HOST(url) │ │ true ║ + *╟─────────────────────────────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┼─────────────────────────────────────────────────────┼───────╢ + *║ Apache Metron/PARSER/foo/fieldTransformations/0/Field Mapping/domain_without_subdomains │ DOMAIN_REMOVE_SUBDOMAINS(full_hostname) │ │ true ║ + *╚═════════════════════════════════════════════════════════════════════════════════════════╧══════════════════════════════════════════╧═════════════════════════════════════════════════════╧═══════╝ + **/ + @Multiline + private static String expectedOutput; + + + /** + *╔════════════════════════════════════════════════╤══════════════════════════════════════════╤═════════════════════╤═══════╗ + *║ PATH │ RULE │ ERROR │ VALID ║ + *╠════════════════════════════════════════════════╪══════════════════════════════════════════╪═════════════════════╪═══════╣ + *║ Apache │ URL_TO_HOST(url)= │ Syntax error @ 1:16 │ false ║ + *║ Metron/PARSER/bar/fieldTransformations/0/Field │ │ token recognition │ ║ + *║ Mapping/full_hostname │ │ error at: '=' │ ║ + *╟────────────────────────────────────────────────┼──────────────────────────────────────────┼─────────────────────┼───────╢ + *║ Apache │ DOMAIN_REMOVE_SUBDOMAINS(full_hostname)= │ Syntax error @ 1:39 │ false ║ + *║ Metron/PARSER/bar/fieldTransformations/0/Field │ │ token recognition │ ║ + *║ Mapping/domain_without_subdomains │ │ error at: '=' │ ║ + *╟────────────────────────────────────────────────┼──────────────────────────────────────────┼─────────────────────┼───────╢ + *║ Apache │ URL_TO_HOST(url) │ │ true ║ + *║ Metron/PARSER/foo/fieldTransformations/0/Field │ │ │ ║ + *║ Mapping/full_hostname │ │ │ ║ + *╟────────────────────────────────────────────────┼──────────────────────────────────────────┼─────────────────────┼───────╢ + *║ Apache │ DOMAIN_REMOVE_SUBDOMAINS(full_hostname) │ │ true ║ + *║ Metron/PARSER/foo/fieldTransformations/0/Field │ │ │ ║ + *║ Mapping/domain_without_subdomains │ │ │ ║ + *╚════════════════════════════════════════════════╧══════════════════════════════════════════╧═════════════════════╧═══════╝ + */ + @Multiline + private static String expectedWrappedOutput; + + @BeforeClass + public static void setupZookeeper() throws Exception { + Properties properties = new Properties(); + zkServerComponent = getZKServerComponent(properties); + zkServerComponent.start(); + } + + @AfterClass + public static void tearDownZookeeper() { + zkServerComponent.stop(); + } + + + @Test + public void test() throws Exception { + try( CuratorFramework client = ConfigurationsUtils.getClient(zkServerComponent.getConnectionString())) { + client.start(); + setupConfiguration(client); + Context context = new Context.Builder().with(ZOOKEEPER_CLIENT, () -> client).build(); + Object out = run("VALIDATE_STELLAR_RULE_CONFIGS()", new HashMap<>(), context); + Assert.assertEquals(expectedOutput, out); + } + + } + @Test + public void testWrapped() throws Exception { + try( CuratorFramework client = ConfigurationsUtils.getClient(zkServerComponent.getConnectionString())) { + client.start(); + setupConfiguration(client); + Context context = new Context.Builder().with(ZOOKEEPER_CLIENT, () -> client).build(); + Object out = run("VALIDATE_STELLAR_RULE_CONFIGS(20)", new HashMap<>(), context); + Assert.assertEquals(expectedWrappedOutput, out); + } + + } + private void setupConfiguration(CuratorFramework client) throws Exception { + + // upload squid configuration to zookeeper + ConfigurationsUtils.writeSensorParserConfigToZookeeper("foo", + config.getBytes(), client); + + ConfigurationsUtils.writeSensorParserConfigToZookeeper("bar", + badConfig.getBytes(), client); + + } + +} \ No newline at end of file diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 078799e136..b228e8a788 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -1422,7 +1422,6 @@ IS_EMAIL ret: True if the string is a valid email address and false otherwise. [Stellar]>>> ``` - ### Advanced Usage To run the Stellar Shell directly from the Metron source code, run a command like the following. Ensure that Metron has already been built and installed with `mvn clean install -DskipTests`. diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml index dc4eb9043b..4d6b187610 100644 --- a/metron-stellar/stellar-common/pom.xml +++ b/metron-stellar/stellar-common/pom.xml @@ -68,7 +68,7 @@ org.apache.commons commons-lang3 - 3.2 + 3.7 commons-io diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java new file mode 100644 index 0000000000..64ef482d42 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java @@ -0,0 +1,325 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.lang.annotation.IncompleteAnnotationException; +import java.lang.reflect.Field; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang.NullArgumentException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; + +/** + *

+ * {@code ExpressionConfigurationHolder} is a wrapper class for Objects that represent + * configurations that have been annotated with the annotations from the validation.annotations + * package. + *

+ *

+ * Holders understand how to discover the stellar expressions contained in them by understanding how + * to evaluate thier own annotated fields. + *

+ *

+ * No knowledge of the implementation of the configuration + * Object itself is required by this class. Instead it evaluates it's own member fields for + * {@code StellarExpressionField}, {@code StellarExpressionList}, {@code StellarExpressionMap} instances. + *

+ *

+ * Complex types annotated by {@code StellarConfiguration} or {@code StellarConfigurationList} are + * treated as children of this class. + *

+ */ +public class ExpressionConfigurationHolder implements StellarConfiguredStatementContainer { + + private Object holderObject; + private String name; + private String parentName; + private String fullName; + private List children = new LinkedList<>(); + private List expressionList; + private List expressionListList; + private List expressionMapList; + + /** + * Constructs a new {@code ExpressionConfigurationHolder}. + * + * @param parentName the name of the parent object of this holder + * @param name the name of to be used for this holder + * @param holderObject the object being held + */ + public ExpressionConfigurationHolder(String parentName, String name, Object holderObject) { + if (holderObject == null) { + throw new NullArgumentException("holderObject"); + } + if (StringUtils.isEmpty(name)) { + throw new NullArgumentException("name"); + } + this.name = name; + this.parentName = parentName; + + this.fullName = StringUtils.isEmpty(parentName) ? this.name + : String.format("%s/%s", this.parentName, this.name); + + this.holderObject = holderObject; + } + + /** + * Returns the {@code Object} being held. + * + * @return {@code Object} + */ + public Object getHolderObject() { + return holderObject; + } + + /** + * Returns the name for this {@code ExpressionConfigurationHolder}. + * + * @return String value of the name + */ + @Override + public String getName() { + return name; + } + + /** + * Returns the name of the parent of this {@code ExpressionConfigurationHolder}. + * + * @return String value of the name + */ + public String getParentName() { + return parentName; + } + + /** + * Returns the full name of this {@code ExpressionConfigurationHolder}. + * This represents the name taken together with the parentName, path delimeted by a "/" + * + * @return String value of the name + */ + @Override + public String getFullName() { + return fullName; + } + + @Override + public void visit(StatementVisitor visitor, ErrorConsumer errorConsumer) { + visitExpressions(visitor, errorConsumer); + visitLists(visitor, errorConsumer); + visitMaps(visitor, errorConsumer); + visitChilden(visitor, errorConsumer); + } + + private void visitExpressions(StatementVisitor visitor, ErrorConsumer errorConsumer) { + expressionList.forEach((f) -> { + String thisFullName = String + .format("%s/%s", getFullName(), f.getAnnotation(StellarExpressionField.class).name()); + try { + Object thisExpressionObject = FieldUtils.readField(f, holderObject, true); + if (thisExpressionObject == null || StringUtils.isEmpty(thisExpressionObject.toString())) { + return; + } + visitExpression(thisFullName, thisExpressionObject.toString(), visitor, errorConsumer); + } catch (IllegalAccessException e) { + errorConsumer.consume(thisFullName, e); + } + }); + } + + private void visitExpression(String expressionName, String expression, StatementVisitor visitor, + ErrorConsumer errorConsumer) { + if (StringUtils.isEmpty(expression)) { + return; + } + try { + visitor.visit(expressionName, expression); + } catch (Exception e) { + errorConsumer.consume(expressionName, e); + } + } + + private void visitLists(StatementVisitor visitor, ErrorConsumer errorConsumer) { + expressionListList.forEach((l) -> { + String thisFullName = String + .format("%s/%s", getFullName(), l.getAnnotation(StellarExpressionList.class).name()); + try { + Object possibleIterable = FieldUtils.readField(l, holderObject, true); + if (possibleIterable == null) { + return; + } + Iterable it = (Iterable) possibleIterable; + int index = 0; + for (Object expressionObject : it) { + String expressionFullName = String.format("%s/%s", thisFullName, index); + visitExpression(expressionFullName, expressionObject.toString(), visitor, errorConsumer); + index++; + } + } catch (IllegalAccessException e) { + errorConsumer.consume(thisFullName, e); + } + }); + } + + @SuppressWarnings("unchecked") + private void visitMaps(StatementVisitor visitor, ErrorConsumer errorConsumer) { + expressionMapList.forEach((f) -> { + String thisFullName = String + .format("%s/%s", getFullName(), f.getAnnotation(StellarExpressionMap.class).name()); + try { + // if we have configured a qualifying field as a type of flag we need to check it + // for example, if StellarFoo.class.isAssignableFrom(this.foo.getClass()) then + // bar is a StellarExpressionMap + if (!StringUtils + .isEmpty(f.getAnnotation(StellarExpressionMap.class).qualifyWithField())) { + String fieldName = f.getAnnotation(StellarExpressionMap.class).qualifyWithField(); + Class type = f.getAnnotation(StellarExpressionMap.class).qualifyWithFieldType(); + Object theObject = FieldUtils.readField(holderObject, fieldName, true); + if (theObject == null) { + errorConsumer.consume(thisFullName, + new IncompleteAnnotationException(StellarExpressionMap.class, "fieldName")); + return; + } + if (!type.isAssignableFrom(theObject.getClass())) { + return; + } + } + + Map map = (Map) FieldUtils.readField(f, holderObject, true); + + // some maps actually nest the config, so check and dig to get the real map + String[] innerKeys = f.getAnnotation(StellarExpressionMap.class).innerMapKeys(); + if (innerKeys.length != 0) { + for (String key : innerKeys) { + if (StringUtils.isEmpty(key)) { + return; + } + Object innerObject = map.get(key); + if (innerObject == null) { + return; + } + thisFullName = String.format("%s/%s", thisFullName, key); + if (!Map.class.isAssignableFrom(innerObject.getClass())) { + errorConsumer.consume(thisFullName, + new Exception("The annotation specified an inner map that was not a map")); + } + map = (Map) innerObject; + } + } + + final String finalName = thisFullName; + map.forEach((k, v) -> { + String mapKeyFullName = String.format("%s/%s", finalName, k.toString()); + if (Map.class.isAssignableFrom(v.getClass())) { + Map innerMap = (Map) v; + innerMap.forEach((ik, iv) -> { + if (iv == null) { + return; + } + visitExpression(String.format("%s/%s", mapKeyFullName, ik.toString()), iv.toString(), + visitor, errorConsumer); + }); + return; + } + visitExpression(mapKeyFullName, v.toString(), visitor, errorConsumer); + }); + } catch (IllegalAccessException e) { + errorConsumer.consume(thisFullName, e); + } + }); + } + + private void visitChilden(StatementVisitor visitor, ErrorConsumer errorConsumer) { + children.forEach((c) -> c.visit(visitor, errorConsumer)); + } + + /** + * When {@code discover} is called, the {@code ExpressionConfigurationHolder} evaluates it's + * members such that it discovers the fields annotated with any of the validation.annotations. + *

+ * The annotated objects found are grouped by annotation type. Any objects annotated as + * {@code StellarConfiguration} will have new {@code ExpressionConfigurationHolder} objects + * created for them, and these objects will be added as children of this + * {@code ExpressionConfigurationHolder}. + *

+ *

+ * Discovery uses the {@code org.apache.commons.lang3.reflect.FieldUtils} classes for finding + * and reading the values of annotated fields. + * This class uses fields as opposed to methods to avoid an possible mutation or side effects + * from the discovery operation. + *

+ * The same for the contents of an object annotated as a {@code StellarConfigurationList}. + */ + @Override + public void discover() throws Exception { + expressionList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionField.class); + expressionListList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionList.class); + expressionMapList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionMap.class); + List holderList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarConfiguration.class); + List holderListList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarConfigurationList.class); + + for (Field f : holderList) { + Object potentialChild = FieldUtils.readField(f, holderObject, true); + if (potentialChild == null) { + break; + } + + ExpressionConfigurationHolder child = new ExpressionConfigurationHolder(getFullName(), + f.getName(), potentialChild); + + child.discover(); + children.add(child); + } + + for (Field f : holderListList) { + String thisFullName = String + .format("%s/%s", getFullName(), f.getAnnotation(StellarConfigurationList.class).name()); + Object potentialChild = FieldUtils.readField(f, holderObject, true); + if (potentialChild == null) { + break; + } + if (!Iterable.class.isAssignableFrom(potentialChild.getClass())) { + break; + } + Iterable it = (Iterable) FieldUtils.readField(f, holderObject, true); + int index = 0; + for (Object thisHolderObject : it) { + if (thisHolderObject == null) { + break; + } + ExpressionConfigurationHolder child = new ExpressionConfigurationHolder(thisFullName, + String.valueOf(index), thisHolderObject); + index++; + child.discover(); + children.add(child); + } + } + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java new file mode 100644 index 0000000000..b182d54d8b --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.util.List; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer; +import org.atteo.classindex.IndexSubclasses; + +/** + * {@code StellarConfigurationProvider} are used provide Stellar statements + * and the context around those statements to the caller. + */ +@IndexSubclasses +public interface StellarConfigurationProvider { + /** + * The Name of this reporter. + * @return String + */ + String getName(); + + /** + * Returns a list of all known StellarConfiguredStatementContainer. + * @return List of StellarConfiguredStatementContainer + */ + List provideContainers(ErrorConsumer errorConsumer); + +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java new file mode 100644 index 0000000000..4fe7065b64 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +/** + * StellarConfiguredStatementProviders are used provide stellar statements + * and the context around those statements to the caller. + * + * The usage pattern is: + * + * stellarConfiguredStatementContainer.discover(); + * stellarConfiguredStatementContiner.visit((p,s)->{}, (p,e) -> {} ); + * + */ +public interface StellarConfiguredStatementContainer { + + interface StatementVisitor { + void visit(String path, String statement); + } + + interface ErrorConsumer { + void consume(String path, Exception e); + } + + /** + * Returns a name for this container. + * @return String + */ + String getName(); + + /** + * Returns a full name for this container. + * This would include pathing information if applicable. + * @return String + */ + String getFullName(); + + /** + * Discover is called to allow the Container to discover it's configurations. + * @throws Exception if there is an error + */ + void discover() throws Exception; + + /** + * Visit each configuration within this container. + * @param visitor StatementVisitor callback + * @param errorConsumer ErrorConsumer callback + * @throws Exception if there is an error + */ + void visit(StatementVisitor visitor, ErrorConsumer errorConsumer) throws Exception; + + +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java new file mode 100644 index 0000000000..69eaba29d0 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java @@ -0,0 +1,26 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation; + + +public interface StellarValidator { + Iterable validate(); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperConfigurationProvider.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperConfigurationProvider.java new file mode 100644 index 0000000000..cc3244e946 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperConfigurationProvider.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.util.List; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer; +import org.atteo.classindex.IndexSubclasses; + +/** + * {@code StellarZookeeperConfigurationProvider} are used provide Stellar statements + * and the context around those statements to the caller. + */ +@IndexSubclasses +public interface StellarZookeeperConfigurationProvider { + + /** + * The Name of this reporter. + * @return String + */ + String getName(); + + /** + * Returns a list of all known StellarConfiguredStatementContainer. + * @param client The Zookeeper client + * @return List of StellarConfiguredStatementContainer + */ + List provideContainers(CuratorFramework client, ErrorConsumer errorConsumer); + +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ValidationResult.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ValidationResult.java new file mode 100644 index 0000000000..b26636bd08 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ValidationResult.java @@ -0,0 +1,80 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +/** + * Result of a validation of a Stellar Rule + */ +public class ValidationResult { + + private String rulePath; + private String rule; + private String error; + private boolean valid; + + public ValidationResult(String rulePath, String rule, String error, boolean valid) { + this.rulePath = rulePath; + this.rule = rule; + this.error = error; + this.valid = valid; + } + + public String getRulePath() { + return rulePath; + } + + public String getRule() { + return rule; + } + + public String getError() { + return error; + } + + public boolean isValid() { + return valid; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + ValidationResult that = (ValidationResult) o; + + return new EqualsBuilder().append(isValid(), that.isValid()).append(getRule(), that.getRule()) + .append(getError(), that.getError()).isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37).append(getRule()).append(getError()).append(isValid()) + .toHashCode(); + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java new file mode 100644 index 0000000000..3a861dd682 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * {@code StellarConfiguration} marks a field or class has a container + * for some stellar configuration or expression. + * + * That expression may be in a field such as a String, or a Map, or list or other complex type. + * Each of those types in turn should be annotated with the appropriate annotation. + * + * Other complex types that are fields of a {@code StellarConfiguration} , should likewise be annotated as + * {@code StellarConfiguration}. + * + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) +public @interface StellarConfiguration { +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java new file mode 100644 index 0000000000..067695ef14 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * {@code StellarConfigurationList} marks a field as being a list + * of objects that represent {@code StellarConfiguration} annotated objects. + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) +public @interface StellarConfigurationList { + /** + * The Name to be applied to this field. + * @return the Name + */ + String name(); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java new file mode 100644 index 0000000000..c656787ab2 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * {@code StellarExpressionField} marks a field + * as being a Stellar expression. + * + * It's toString() method will be called to provide this String. + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) +public @interface StellarExpressionField { + + /** + * The Name to be applied to this field. + * @return the Name + */ + String name(); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java new file mode 100644 index 0000000000..b19292dfd5 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * {@code StellarExpressionList} is applied to a field + * that is an {@code Iterable} of objects that are Stellar expressions, + * Such that calling toString() on that object yields a string of a Stellar expression + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) +public @interface StellarExpressionList { + /** + * The Name to be applied to this field. + * @return the Name + */ + String name(); + +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java new file mode 100644 index 0000000000..01e898bea7 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * {@code StellarExpressionMap} is applied to + * a {@code Map} which contains Stellar expressions as the values, such + * that calling .toString() on a value of this map yields a Stellar expression. + * + * The key is used as the name the expression. + * + * It is possible for a {@code Map} to contain other maps or complex objects, + * thus this annotation contains properties that give information on evaluation of the {@code Map} + * should it be complex and contain nested maps. + * + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) +public @interface StellarExpressionMap { + + /** + * The Name to give to the map + * + * @return String of the name + */ + String name() default "default"; + + /** + * A map may be a StellarExpressionMap based on the type + * of another field, this is that field's name. + * + * {@code} qualifyWithFieldType} is the type of that field + * + * @return Field Name or empty String + */ + String qualifyWithField() default ""; + + /** + * A map may be a StellarExpressionMap based on the type + * of another field, this is that type of that field. + * + * {@code qualifyWithField} is the name of the field. + * + * @return Class + */ + Class qualifyWithFieldType() default Error.class; + + + /** + * Some maps, may 'hold' the real map as a value. In that case the outer map should still be + * annotated, and this property used to define the 'key route' needed to obtain the map. + * These are the key to get that map. + * There are multiple in case there are multiple nested level + * This does not support multiple nested 'peer' maps + * + * @return String key + */ + String[] innerMapKeys() default {}; + +} \ No newline at end of file diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java new file mode 100644 index 0000000000..cddddc0033 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java @@ -0,0 +1,68 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation.validators; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; +import org.apache.metron.stellar.common.utils.validation.StellarValidator; +import org.apache.metron.stellar.common.utils.validation.ValidationResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BaseValidator implements StellarValidator { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String FAILED_COMPILE = "Failed to compile"; + + public BaseValidator() { + } + + @Override + public abstract Iterable validate(); + + protected List handleContainers( + List containers) { + ArrayList results = new ArrayList<>(); + containers.forEach((container) -> { + try { + container.discover(); + container.visit((path, statement) -> { + try { + if (StellarProcessor.compile(statement) == null) { + results.add(new ValidationResult(path, statement, FAILED_COMPILE, false)); + } else { + results.add(new ValidationResult(path, statement, null, true)); + } + } catch (RuntimeException e) { + results.add(new ValidationResult(path, statement, e.getMessage(), false)); + } + }, (path, error) -> { + results.add(new ValidationResult(path, null, error.getMessage(), false)); + }); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + }); + return results; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarSimpleValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarSimpleValidator.java new file mode 100644 index 0000000000..c0fba30661 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarSimpleValidator.java @@ -0,0 +1,77 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation.validators; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.utils.validation.StellarConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; +import org.apache.metron.stellar.common.utils.validation.StellarValidator; +import org.apache.metron.stellar.common.utils.validation.StellarZookeeperConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.ValidationResult; +import org.atteo.classindex.ClassIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StellarSimpleValidator extends BaseValidator { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String FAILED_COMPILE = "Failed to compile"; + + public StellarSimpleValidator(){} + + @Override + public Iterable validate() { + // discover all the StellarConfigurationProvider + Set providerSet = new HashSet<>(); + + for (Class c : ClassIndex.getSubclasses(StellarZookeeperConfigurationProvider.class, + Thread.currentThread().getContextClassLoader())) { + boolean isAssignable = StellarConfigurationProvider.class.isAssignableFrom(c); + if (isAssignable) { + try { + StellarConfigurationProvider reporter = StellarConfigurationProvider.class + .cast(c.getConstructor().newInstance()); + providerSet.add(reporter); + } catch (Exception e) { + LOG.error("Provider: " + c.getCanonicalName() + " not valid, skipping", e); + } + } + } + + ArrayList results = new ArrayList<>(); + providerSet.forEach((r) -> { + try { + List containers = r + .provideContainers((pathName, exception) -> { + results.add(new ValidationResult(pathName, null, exception.getMessage(), false)); + }); + results.addAll(handleContainers(containers)); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + }); + return results; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarZookeeperBasedValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarZookeeperBasedValidator.java new file mode 100644 index 0000000000..d564e47208 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarZookeeperBasedValidator.java @@ -0,0 +1,85 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation.validators; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.commons.lang.NullArgumentException; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; +import org.apache.metron.stellar.common.utils.validation.StellarValidator; +import org.apache.metron.stellar.common.utils.validation.StellarZookeeperConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.ValidationResult; +import org.atteo.classindex.ClassIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StellarZookeeperBasedValidator extends BaseValidator { + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private CuratorFramework client; + + public StellarZookeeperBasedValidator(CuratorFramework client) throws NullArgumentException { + if (client == null) { + throw new NullArgumentException("client"); + } + this.client = client; + } + + + @Override + public Iterable validate() { + // discover all the StellarZookeeperConfigurationProvider + Set providerSet = new HashSet<>(); + + for (Class c : ClassIndex.getSubclasses(StellarZookeeperConfigurationProvider.class, + Thread.currentThread().getContextClassLoader())) { + boolean isAssignable = StellarZookeeperConfigurationProvider.class.isAssignableFrom(c); + if (isAssignable) { + try { + StellarZookeeperConfigurationProvider reporter = StellarZookeeperConfigurationProvider.class + .cast(c.getConstructor().newInstance()); + providerSet.add(reporter); + } catch (Exception e) { + LOG.error("Provider: " + c.getCanonicalName() + " not valid, skipping", e); + } + } + } + + ArrayList results = new ArrayList<>(); + providerSet.forEach((r) -> { + try { + List containers = r + .provideContainers(client, (pathName, exception) -> { + results.add(new ValidationResult(pathName, null, exception.getMessage(), false)); + }); + results.addAll(handleContainers(containers)); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + }); + return results; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java index 5acb42ce94..642229fcc5 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java @@ -17,6 +17,7 @@ */ package org.apache.metron.stellar.dsl.functions.resolver; +import java.util.List; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.StellarFunction; import org.apache.metron.stellar.dsl.StellarFunctionInfo; diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolderTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolderTest.java new file mode 100644 index 0000000000..dc6e62a303 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolderTest.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.util.LinkedList; +import java.util.List; +import org.junit.Assert; +import org.junit.Test; + +public class ExpressionConfigurationHolderTest { + + @Test + @SuppressWarnings("unchecked") + public void visitAndDiscover() throws Exception { + List discoveredPaths = new LinkedList<>(); + List expectedPaths = new LinkedList(){{ + add("unit tests/test config/the field"); + add("unit tests/test config/the list of stellar strings/0"); + add("unit tests/test config/the list of stellar strings/1"); + add("unit tests/test config/plain old map/stellar_map_item"); + add("unit tests/test config/map two deep/level1/level2/nested stellar key"); + add("unit tests/test config/map depending on field type/stellar_key"); + add("unit tests/test config/child/child field"); + add("unit tests/test config/list of holders/0/child field"); + add("unit tests/test config/list of holders/1/child field"); + }}; + TestConfigObject testSubject = new TestConfigObject(); + ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder("unit tests", + "test config", testSubject); + + holder.discover(); + holder.visit(((path, statement) -> { + discoveredPaths.add(path); + }),((path, e) -> { + Assert.assertTrue("Should not get errors", false); + })); + + Assert.assertFalse(discoveredPaths.contains("unit tests/test config/map depending on field type but wrong/stellar_key")); + + Assert.assertEquals(discoveredPaths.size(),expectedPaths.size()); + expectedPaths.forEach((p) -> { + if(!discoveredPaths.contains(p)) { + Assert.assertTrue(String.format("discovered missing %s", p), false); + } + }); + + discoveredPaths.removeAll(expectedPaths); + + Assert.assertTrue(discoveredPaths.isEmpty()); + } + +} \ No newline at end of file diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestChildConfigObject.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestChildConfigObject.java new file mode 100644 index 0000000000..9318bea949 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestChildConfigObject.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; + +@StellarConfiguration +public class TestChildConfigObject { + @StellarExpressionField(name = "child field") + private String field; + + public TestChildConfigObject(Number number){ + field = String.format("TO_STRING(%d)",number.intValue()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java new file mode 100644 index 0000000000..004821ddba --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; + +@StellarConfiguration +@SuppressWarnings("unchecked") +public class TestConfigObject { + + // a field + @StellarExpressionField(name = "the field") + private String stellar = "TO_UPPER('stellarexpressionfield')"; + + // a list of stellar + @StellarExpressionList(name = "the list of stellar strings") + private List stellarList = new LinkedList() {{ + add("TO_UPPER('list stellar item one')"); + add("TO_UPPER('list stellar item two')"); + }}; + + + // a map + @StellarExpressionMap(name = "plain old map") + private Map plainMap = new HashMap() {{ + put("stellar_map_item", "TO_UPPER('SMI')"); + }}; + + + // a map with the 'real' map burried 2 layers in + @StellarExpressionMap(name = "map two deep", innerMapKeys = {"level1", "level2"}) + private Map mapTwoDeep = new HashMap(){{ + put("level1",new HashMap(){{ + put("level2",new HashMap(){{ + put("nested stellar key", "TO_UPPER('dig down deep')"); + }}); + }}); + }}; + + @StellarExpressionMap(name = "map depending on field type", qualifyWithField = "qualifier_field", qualifyWithFieldType = java.lang.Integer.class) + private Map dependsMap = new HashMap(){{ + put("stellar_key", "TO_UPPER('it was int')"); + }}; + + @StellarExpressionMap(name = "map depending on field type but wrong", qualifyWithField = "qualifier_field", qualifyWithFieldType = java.lang.String.class) + private Map dependsButNotMap = new HashMap(){{ + put("stellar_key", "TO_UPPER('it was not int')"); + }}; + + private Object qualifier_field = new Integer(2); + + @StellarConfiguration + private TestChildConfigObject child = new TestChildConfigObject(1); + + @StellarConfigurationList(name = "list of holders") + private List stellarConfigList = new LinkedList(){{ + add(new TestChildConfigObject(2)); + add(new TestChildConfigObject(3)); + }}; +}