From a5087f3a170eeda6ee778397c919d9eddd5597e2 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Thu, 30 Nov 2017 09:15:40 -0500 Subject: [PATCH 01/14] Stellar shell functionality to verify stellar statements. This will allow users to check their deployed statements, say after upgrade, when they are at rest ( and would fail on use ). In other words, they were valid when stored, but are not now because of stellar changes, such as new keywords. The interface StellarConfiguredStatementReporter, which is @IndexSubclasses marked, allows the shell to discover reporters that can provide statements for validation. This discovery allows de-coupling of stellar and 'hosts' that know about the location of the stored statements, and the configuration structure details. We do mention the configurations in the shell output at this time. metron-common implements this interface, and can run through visiting all the configurations. --- .../StellarStatementReporter.java | 166 ++++++++++++++++++ .../StellarConfiguredStatementReporter.java | 48 +++++ .../stellar/common/shell/StellarExecutor.java | 5 + .../stellar/common/shell/StellarShell.java | 92 +++++++++- .../functions/resolver/FunctionResolver.java | 1 + 5 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarConfiguredStatementReporter.java diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java new file mode 100644 index 0000000000..6ae711c8a9 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.metron.common.configuration; + +import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; +import static org.apache.metron.common.configuration.ConfigurationType.PARSER; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; +import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; +import org.apache.metron.common.field.transformation.FieldTransformation; +import org.apache.metron.common.field.transformation.StellarTransformation; +import org.apache.metron.common.utils.StringUtils; +import org.apache.metron.stellar.common.StellarConfiguredStatementReporter; + +/** + * StellarStatementReporter is used to report all of the configured / deployed Stellar statements in + * the system. + */ +public class StellarStatementReporter implements StellarConfiguredStatementReporter { + + public enum Type { + ENRICHMENT, THREAT_INTEL; + } + + public StellarStatementReporter() { + } + + @Override + public String getName() { + return "Apache Metron"; + } + + @Override + public void vist(CuratorFramework client, StatementReportVisitor visitor, + ConfigReportErrorConsumer errorConsumer) throws Exception { + visitParserConfigs(client, visitor, errorConsumer); + visitEnrichmentConfigs(client, visitor, errorConsumer); + } + + private void visitParserConfigs(CuratorFramework client, StatementReportVisitor visitor, + ConfigReportErrorConsumer errorConsumer) throws Exception { + List children = client.getChildren().forPath(PARSER.getZookeeperRoot()); + for (String child : children) { + byte[] data = client.getData().forPath(PARSER.getZookeeperRoot() + "/" + child); + try { + SensorParserConfig parserConfig = SensorParserConfig.fromBytes(data); + List transformations = parserConfig.getFieldTransformations(); + transformations.forEach((f) -> { + if (StellarTransformation.class.isAssignableFrom(f.getFieldTransformation().getClass())) { + FieldTransformation transformation = f.getFieldTransformation(); + f.getConfig().forEach((k, v) -> { + List names = Arrays + .asList(getName(), PARSER.toString(), parserConfig.getSensorTopic(), k); + visitor.visit(names, v.toString()); + }); + } + }); + } catch (Exception e) { + List names = Arrays.asList(getName(), PARSER.toString(), child); + errorConsumer.consume(names, e); + } + } + } + + @SuppressWarnings("unchecked") + private void visitEnrichmentConfigs(CuratorFramework client, StatementReportVisitor visitor, + ConfigReportErrorConsumer errorConsumer) throws Exception { + List children = client.getChildren().forPath(ENRICHMENT.getZookeeperRoot()); + for (String child : children) { + byte[] data = client.getData().forPath(ENRICHMENT.getZookeeperRoot() + "/" + child); + try { + final SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig + .fromBytes(data); + + EnrichmentConfig enrichmentConfig = null; + enrichmentConfig = sensorEnrichmentConfig.getEnrichment(); + visitEnrichementConfig(child, Type.ENRICHMENT, enrichmentConfig, visitor, errorConsumer); + enrichmentConfig = sensorEnrichmentConfig.getThreatIntel(); + visitEnrichementConfig(child, Type.THREAT_INTEL, enrichmentConfig, visitor, errorConsumer); + ThreatTriageConfig threatTriageConfig = sensorEnrichmentConfig.getThreatIntel() + .getTriageConfig(); + visitEnrichmentThreatTriageConfigs(child, threatTriageConfig, visitor, errorConsumer); + } catch (Exception e) { + List names = Arrays.asList(getName(), ENRICHMENT.toString(), child); + errorConsumer.consume(names, e); + } + } + } + + @SuppressWarnings("unchecked") + private void visitEnrichementConfig(String topicName, Type type, + EnrichmentConfig enrichmentConfig, StatementReportVisitor visitor, + ConfigReportErrorConsumer errorConsumer) throws Exception { + + Map enrichmentStellarMap = (Map) enrichmentConfig.getFieldMap() + .getOrDefault("stellar", new HashMap<>()); + Map transforms = (Map) enrichmentStellarMap + .getOrDefault("config", new HashMap<>()); + try { + for (Map.Entry kv : transforms.entrySet()) { + // we can have a group or an entry + if (kv.getValue() instanceof Map) { + Map groupMap = (Map) kv.getValue(); + for (Map.Entry groupKv : groupMap.entrySet()) { + List names = Arrays + .asList(getName(), ENRICHMENT.toString(), topicName, type.toString(), kv.getKey(), + groupKv.getKey()); + visitor.visit(names, groupKv.getValue()); + } + } else { + List names = Arrays + .asList(getName(), ENRICHMENT.toString(), topicName, type.toString(), "(default)", + kv.getKey(), kv.getKey()); + visitor.visit(names, kv.getValue().toString()); + } + } + } catch (Exception e) { + List names = Arrays + .asList(getName(), ENRICHMENT.toString(), topicName, type.toString()); + errorConsumer.consume(names, e); + } + } + + @SuppressWarnings("unchecked") + private void visitEnrichmentThreatTriageConfigs(String topicName, + ThreatTriageConfig threatTriageConfig, StatementReportVisitor visitor, + ConfigReportErrorConsumer errorConsumer) throws Exception { + try { + List riskLevelRules = threatTriageConfig.getRiskLevelRules(); + riskLevelRules.forEach((r) -> { + String name = r.getName(); + if (org.apache.commons.lang.StringUtils.isEmpty(name)) { + name = "(default)"; + } + List names = Arrays + .asList(getName(), ENRICHMENT.toString(), topicName, "THREAT_TRIAGE", name); + visitor.visit(names, r.getRule()); + }); + + } catch (Exception e) { + List names = Arrays + .asList(getName(), ENRICHMENT.toString(), topicName, "THREAT_TRIAGE"); + errorConsumer.consume(names, e); + } + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarConfiguredStatementReporter.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarConfiguredStatementReporter.java new file mode 100644 index 0000000000..aafd544d2f --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarConfiguredStatementReporter.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common; + +import java.util.List; +import org.apache.curator.framework.CuratorFramework; +import org.atteo.classindex.IndexSubclasses; + +/** + * StellarConfiguredStatementProviders are used provide stellar statements + * and the context around those statements to the caller + */ +@IndexSubclasses +public interface StellarConfiguredStatementReporter { + + /** + * The Name of this reporter + * @return String + */ + String getName(); + + public interface StatementReportVisitor{ + void visit(List contextNames, String statement); + } + + public interface ConfigReportErrorConsumer { + void consume(List contextNames, Exception e); + } + + void vist(CuratorFramework client, StatementReportVisitor visitor, ConfigReportErrorConsumer errorConsumer) throws Exception; + +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java index 1be38c3673..347245009f 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java @@ -208,6 +208,7 @@ private PatriciaTrie initializeIndex() { index.put(StellarShell.MAGIC_GLOBALS, AutoCompleteType.FUNCTION); index.put(StellarShell.MAGIC_DEFINE, AutoCompleteType.FUNCTION); index.put(StellarShell.MAGIC_UNDEFINE, AutoCompleteType.FUNCTION); + index.put(StellarShell.MAGIC_VALIDATE_CONFIGURED, AutoCompleteType.FUNCTION); return new PatriciaTrie<>(index); } @@ -323,5 +324,9 @@ public FunctionResolver getFunctionResolver() { public Context getContext() { return context; } + + public Optional getClient() { + return client; + } } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index 6f23a5ebc4..109d434d53 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -24,12 +24,20 @@ import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import java.io.PrintStream; +import java.util.HashSet; +import java.util.Set; import org.apache.commons.cli.*; import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; import org.apache.log4j.PropertyConfigurator; import org.apache.metron.stellar.common.StellarAssignment; +import org.apache.metron.stellar.common.StellarConfiguredStatementReporter; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.configuration.ConfigurationsUtils; import org.apache.metron.stellar.common.utils.JSONUtils; import org.apache.metron.stellar.dsl.StellarFunctionInfo; +import org.atteo.classindex.ClassIndex; import org.jboss.aesh.complete.CompleteOperation; import org.jboss.aesh.complete.Completion; import org.jboss.aesh.console.AeshConsoleCallback; @@ -94,6 +102,7 @@ public class StellarShell extends AeshConsoleCallback implements Completion { public static final String MAGIC_GLOBALS = MAGIC_PREFIX + "globals"; public static final String MAGIC_DEFINE = MAGIC_PREFIX + "define"; public static final String MAGIC_UNDEFINE = MAGIC_PREFIX + "undefine"; + public static final String MAGIC_VALIDATE_CONFIGURED = MAGIC_PREFIX + "validate_configured_expressions"; private StellarExecutor executor; @@ -318,7 +327,8 @@ private void handleMagic(String rawExpression) { } else if(MAGIC_UNDEFINE.equals(command)) { handleMagicUndefine(expression); - + } else if(MAGIC_VALIDATE_CONFIGURED.equals(command)) { + handleValidateConfigured(); } else { writeLine(ERROR_PROMPT + "undefined magic command: " + rawExpression); } @@ -403,6 +413,84 @@ private void handleMagicUndefine(String[] expression) { } } + /** + * Handle a magic '%validate_configured_expressions'. + * This command will attempt to locate and validate through compilation + * all deployed stellar statements in the system. + */ + private void handleValidateConfigured() { + + Optional client = executor.getClient(); + if (!client.isPresent()) { + writeLine(ERROR_PROMPT + "Zookeeper is required to validate deployed stellar statements!"); + writeLine(ERROR_PROMPT + "Please restart the Stellar Shell with the -z parameter!"); + return; + } + + // discover all the StellarConfiguredStatementReporters + Set reporterSet = new HashSet<>(); + + for (Class c : ClassIndex.getSubclasses(StellarConfiguredStatementReporter.class, + Thread.currentThread().getContextClassLoader())) { + boolean isAssignable = StellarConfiguredStatementReporter.class.isAssignableFrom(c); + if (isAssignable) { + try { + StellarConfiguredStatementReporter reporter = StellarConfiguredStatementReporter.class + .cast(c.getConstructor().newInstance()); + reporterSet.add(reporter); + } catch (Exception e) { + writeLine(ERROR_PROMPT + " Reporter: " + c.getCanonicalName() + " not valid, skipping"); + } + } + } + + writeLine(String.format("Discovered %d reporters", reporterSet.size())); + + writeLine("Visiting all configurations. ThreatTriage rules are checked when loading the " + + "configuration, thus an invalid ThreatTriage rule will fail the entire Enrichement Configuration."); + if (reporterSet.size() > 0) { + reporterSet.forEach((r) -> write(r.getName() + " ")); + writeLine(""); + } else { + return; + } + + reporterSet.forEach((r) -> { + writeLine("Visiting " + r.getName()); + try { + r.vist(client.get(), ((contextNames, statement) -> { + // the names taken together are the identifier for this + // statement + String name = String.join("->", contextNames); + + writeLine("\n\n==================================================\n\n"); + writeLine("validating " + name); + try { + if (StellarProcessor.compile(statement) == null) { + writeLine( + ERROR_PROMPT + String.format("Statement: %s is not valid, please review", name)); + writeLine(ERROR_PROMPT + String.format("Statement: %s ", statement)); + } + } catch (RuntimeException e) { + writeLine(ERROR_PROMPT + "Error Visiting " + name); + writeLine(e.getMessage()); + writeLine("--"); + writeLine(ERROR_PROMPT + ": " + statement); + } + writeLine("\n\n=================================================="); + }), (contextNames, exception) -> { + String name = String.join("->", contextNames); + writeLine( + ERROR_PROMPT + String.format("Configuration %s is not valid, please review", name)); + }); + } catch (Exception e) { + writeLine(ERROR_PROMPT + "Error Visiting " + r.getName()); + writeLine(e.getMessage()); + } + }); + writeLine("\nDone validation"); + } + /** * Retrieves the GLOBAL_CONFIG, if it exists. If it does not, it creates the GLOBAL_CONFIG * and adds it to the Stellar execution context. @@ -487,7 +575,7 @@ private boolean isDoc(String expression) { } private void write(String out) { - System.out.print(out); + console.getShell().out().print(out); } private void writeLine(String out) { diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java index 5acb42ce94..642229fcc5 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java @@ -17,6 +17,7 @@ */ package org.apache.metron.stellar.dsl.functions.resolver; +import java.util.List; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.StellarFunction; import org.apache.metron.stellar.dsl.StellarFunctionInfo; From 96df802318d74bf8dfcd3bcae9208f63c3d034f0 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Thu, 30 Nov 2017 11:25:58 -0500 Subject: [PATCH 02/14] add readme and remove some newlines --- metron-stellar/stellar-common/README.md | 4 ++++ .../org/apache/metron/stellar/common/shell/StellarShell.java | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 5fe9ae7207..abf4bfa0da 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -1316,6 +1316,10 @@ IS_EMAIL [Stellar]>>> ``` +#### `%validate_configured_expressions` + +Attempts to validate deployed Stellar expressions by ensuring they compile. This is useful when expressions may have been valid when deployed, but may have been invalidated by a language change after that. + ### Advanced Usage To run the Stellar Shell directly from the Metron source code, run a command like the following. Ensure that Metron has already been built and installed with `mvn clean install -DskipTests`. diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index 109d434d53..84a1d4d353 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -463,7 +463,7 @@ private void handleValidateConfigured() { // statement String name = String.join("->", contextNames); - writeLine("\n\n==================================================\n\n"); + writeLine("=================================================="); writeLine("validating " + name); try { if (StellarProcessor.compile(statement) == null) { @@ -477,7 +477,7 @@ private void handleValidateConfigured() { writeLine("--"); writeLine(ERROR_PROMPT + ": " + statement); } - writeLine("\n\n=================================================="); + writeLine("=================================================="); }), (contextNames, exception) -> { String name = String.join("->", contextNames); writeLine( From dcd55e8f4a72e5c3e694807e13c7eebc53d1860f Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Thu, 30 Nov 2017 18:34:38 -0500 Subject: [PATCH 03/14] add tests for StellarStatementReporter --- .../StellarStatementReporter.java | 154 +++++++++- .../StellarStatementReporterTest.java | 274 ++++++++++++++++++ 2 files changed, 413 insertions(+), 15 deletions(-) create mode 100644 metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java index 6ae711c8a9..fcb745ad5d 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more contributor license * agreements. See the NOTICE file distributed with this work for additional information regarding * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 @@ -17,9 +17,11 @@ import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; import static org.apache.metron.common.configuration.ConfigurationType.PARSER; +import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import org.apache.curator.framework.CuratorFramework; @@ -27,10 +29,15 @@ import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; -import org.apache.metron.common.field.transformation.FieldTransformation; +import org.apache.metron.common.configuration.profiler.ProfileConfig; +import org.apache.metron.common.configuration.profiler.ProfileResult; +import org.apache.metron.common.configuration.profiler.ProfileResultExpressions; +import org.apache.metron.common.configuration.profiler.ProfileTriageExpressions; +import org.apache.metron.common.configuration.profiler.ProfilerConfig; import org.apache.metron.common.field.transformation.StellarTransformation; -import org.apache.metron.common.utils.StringUtils; +import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.stellar.common.StellarConfiguredStatementReporter; +import org.apache.zookeeper.KeeperException.NoNodeException; /** * StellarStatementReporter is used to report all of the configured / deployed Stellar statements in @@ -39,7 +46,7 @@ public class StellarStatementReporter implements StellarConfiguredStatementReporter { public enum Type { - ENRICHMENT, THREAT_INTEL; + ENRICHMENT, THREAT_INTEL } public StellarStatementReporter() { @@ -55,11 +62,19 @@ public void vist(CuratorFramework client, StatementReportVisitor visitor, ConfigReportErrorConsumer errorConsumer) throws Exception { visitParserConfigs(client, visitor, errorConsumer); visitEnrichmentConfigs(client, visitor, errorConsumer); + visitProfilerConfigs(client, visitor, errorConsumer); } private void visitParserConfigs(CuratorFramework client, StatementReportVisitor visitor, ConfigReportErrorConsumer errorConsumer) throws Exception { - List children = client.getChildren().forPath(PARSER.getZookeeperRoot()); + List children = null; + + try { + children = client.getChildren().forPath(PARSER.getZookeeperRoot()); + } catch (NoNodeException nne) { + return; + } + for (String child : children) { byte[] data = client.getData().forPath(PARSER.getZookeeperRoot() + "/" + child); try { @@ -67,7 +82,6 @@ private void visitParserConfigs(CuratorFramework client, StatementReportVisitor List transformations = parserConfig.getFieldTransformations(); transformations.forEach((f) -> { if (StellarTransformation.class.isAssignableFrom(f.getFieldTransformation().getClass())) { - FieldTransformation transformation = f.getFieldTransformation(); f.getConfig().forEach((k, v) -> { List names = Arrays .asList(getName(), PARSER.toString(), parserConfig.getSensorTopic(), k); @@ -85,19 +99,37 @@ private void visitParserConfigs(CuratorFramework client, StatementReportVisitor @SuppressWarnings("unchecked") private void visitEnrichmentConfigs(CuratorFramework client, StatementReportVisitor visitor, ConfigReportErrorConsumer errorConsumer) throws Exception { - List children = client.getChildren().forPath(ENRICHMENT.getZookeeperRoot()); + List children = null; + + try { + children = client.getChildren().forPath(ENRICHMENT.getZookeeperRoot()); + } catch (NoNodeException nne) { + return; + } + for (String child : children) { byte[] data = client.getData().forPath(ENRICHMENT.getZookeeperRoot() + "/" + child); try { + // Certain parts of the SensorEnrichmentConfig do Stellar Verification on their + // own as part of deserialization, where the bean spec will call the setter, which has + // been wired with stellar verification calls. + // + // In cases where those parts of the config are in fact the parts that have invalid + // Stellar statements, we will fail during the JSON load before we get to ANY config + // contained in the SensorEnrichmentConfig. + // + // I have left the code to properly check all the configuration parts for completeness + // on the reporting side ( the report initiator may want to list successful evals), even + // though they can be executed, then they will never fail. final SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig .fromBytes(data); - EnrichmentConfig enrichmentConfig = null; + EnrichmentConfig enrichmentConfig; enrichmentConfig = sensorEnrichmentConfig.getEnrichment(); - visitEnrichementConfig(child, Type.ENRICHMENT, enrichmentConfig, visitor, errorConsumer); + visitEnrichmentConfig(child, Type.ENRICHMENT, enrichmentConfig, visitor, errorConsumer); enrichmentConfig = sensorEnrichmentConfig.getThreatIntel(); - visitEnrichementConfig(child, Type.THREAT_INTEL, enrichmentConfig, visitor, errorConsumer); - ThreatTriageConfig threatTriageConfig = sensorEnrichmentConfig.getThreatIntel() + visitEnrichmentConfig(child, Type.THREAT_INTEL, enrichmentConfig, visitor, errorConsumer); + final ThreatTriageConfig threatTriageConfig = sensorEnrichmentConfig.getThreatIntel() .getTriageConfig(); visitEnrichmentThreatTriageConfigs(child, threatTriageConfig, visitor, errorConsumer); } catch (Exception e) { @@ -108,9 +140,9 @@ private void visitEnrichmentConfigs(CuratorFramework client, StatementReportVisi } @SuppressWarnings("unchecked") - private void visitEnrichementConfig(String topicName, Type type, + private void visitEnrichmentConfig(String topicName, Type type, EnrichmentConfig enrichmentConfig, StatementReportVisitor visitor, - ConfigReportErrorConsumer errorConsumer) throws Exception { + ConfigReportErrorConsumer errorConsumer) { Map enrichmentStellarMap = (Map) enrichmentConfig.getFieldMap() .getOrDefault("stellar", new HashMap<>()); @@ -144,7 +176,7 @@ private void visitEnrichementConfig(String topicName, Type type, @SuppressWarnings("unchecked") private void visitEnrichmentThreatTriageConfigs(String topicName, ThreatTriageConfig threatTriageConfig, StatementReportVisitor visitor, - ConfigReportErrorConsumer errorConsumer) throws Exception { + ConfigReportErrorConsumer errorConsumer) { try { List riskLevelRules = threatTriageConfig.getRiskLevelRules(); riskLevelRules.forEach((r) -> { @@ -153,8 +185,13 @@ private void visitEnrichmentThreatTriageConfigs(String topicName, name = "(default)"; } List names = Arrays - .asList(getName(), ENRICHMENT.toString(), topicName, "THREAT_TRIAGE", name); + .asList(getName(), ENRICHMENT.toString(), topicName, "THREAT_TRIAGE", name, "rule"); visitor.visit(names, r.getRule()); + if (!org.apache.commons.lang.StringUtils.isEmpty(r.getReason())) { + names = Arrays + .asList(getName(), ENRICHMENT.toString(), topicName, "THREAT_TRIAGE", name, "reason"); + visitor.visit(names, r.getReason()); + } }); } catch (Exception e) { @@ -163,4 +200,91 @@ private void visitEnrichmentThreatTriageConfigs(String topicName, errorConsumer.consume(names, e); } } + + private void visitProfilerConfigs(CuratorFramework client, StatementReportVisitor visitor, + ConfigReportErrorConsumer errorConsumer) { + try { + byte[] profilerConfigData = null; + try { + profilerConfigData = client.getData().forPath(PROFILER.getZookeeperRoot()); + } catch (NoNodeException nne) { + return; + } + + ProfilerConfig profilerConfig = JSONUtils.INSTANCE + .load(new String(profilerConfigData), ProfilerConfig.class); + profilerConfig.getProfiles().forEach((ProfileConfig pc) -> { + List names = new LinkedList<>(); + names.add(getName()); + names.add(PROFILER.toString()); + names.add(pc.getProfile()); + + // only if + if (!org.apache.commons.lang.StringUtils.isEmpty(pc.getOnlyif())) { + names.add("only_if"); + visitor.visit(names, pc.getOnlyif()); + names.remove("only_if"); + } + + // init + if (!pc.getInit().isEmpty()) { + names.add("init"); + pc.getInit().forEach((k, v) -> { + names.add(k); + visitor.visit(names, v); + names.remove(k); + }); + names.remove("init"); + } + + // update + if (!pc.getUpdate().isEmpty()) { + names.add("update"); + pc.getUpdate().forEach((k, v) -> { + names.add(k); + visitor.visit(names, v); + names.remove(k); + }); + names.remove("update"); + } + + // group by + if (!pc.getGroupBy().isEmpty()) { + names.add("group_by"); + pc.getGroupBy().forEach((gb) -> visitor.visit(names, gb)); + names.remove("group_by"); + } + + // profile result + if (pc.getResult() != null) { + ProfileResult result = pc.getResult(); + ProfileResultExpressions profileResultExpressions = result.getProfileExpressions(); + ProfileTriageExpressions profileTriageExpressions = result.getTriageExpressions(); + + names.add("profile_result"); + if (profileResultExpressions != null && !org.apache.commons.lang.StringUtils + .isEmpty(profileResultExpressions.getExpression())) { + names.add("profile_expression"); + visitor.visit(names, profileResultExpressions.getExpression()); + names.remove("profile_expression"); + } + + if (profileTriageExpressions != null && !profileTriageExpressions.getExpressions() + .isEmpty()) { + names.add("triage_expression"); + profileTriageExpressions.getExpressions().forEach((k, v) -> { + names.add(k); + visitor.visit(names, v); + names.remove(k); + }); + } + } + + + }); + } catch (Exception e) { + List names = Arrays.asList(getName(), PROFILER.toString()); + errorConsumer.consume(names, e); + } + } } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java new file mode 100644 index 0000000000..36deaa3793 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java @@ -0,0 +1,274 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.configuration; + +import java.util.LinkedList; +import java.util.List; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.test.TestingServer; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class StellarStatementReporterTest { + + private TestingServer testZkServer; + private String zookeeperUrl; + private CuratorFramework client; + private List expected; + + @Before + public void setUp() throws Exception { + testZkServer = new TestingServer(true); + zookeeperUrl = testZkServer.getConnectString(); + client = ConfigurationsUtils.getClient(zookeeperUrl); + expected = new LinkedList<>(); + expected.add("Apache Metron->PARSER->squid->full_hostname"); + expected.add("Apache Metron->PARSER->squid->domain_without_subdomains"); + expected.add("Apache Metron->ENRICHMENT->snort->THREAT_TRIAGE->(default)->rule"); + expected.add("Apache Metron->PROFILER->example2->only_if"); + expected.add("Apache Metron->PROFILER->example2->init->num_dns"); + expected.add("Apache Metron->PROFILER->example2->init->num_http"); + expected.add("Apache Metron->PROFILER->example2->update->num_dns"); + expected.add("Apache Metron->PROFILER->example2->update->num_http"); + expected.add("Apache Metron->PROFILER->example2->profile_result->profile_expression"); + expected.add("Apache Metron->PROFILER->example1->only_if"); + expected.add("Apache Metron->PROFILER->example1->init->total_bytes"); + expected.add("Apache Metron->PROFILER->example1->update->total_bytes"); + expected.add("Apache Metron->PROFILER->example1->profile_result->profile_expression"); + expected.add("Apache Metron->PROFILER->percentiles->only_if"); + expected.add("Apache Metron->PROFILER->percentiles->init->s"); + expected.add("Apache Metron->PROFILER->percentiles->update->s"); + expected.add("Apache Metron->PROFILER->percentiles->profile_result->profile_expression"); + client.start(); + } + + @After + public void tearDown() throws Exception { + client.close(); + testZkServer.close(); + testZkServer.stop(); + } + + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "squid", + * "parserConfig": { + * "grokPath": "/patterns/squid", + * "patternLabel": "SQUID_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR", + * "output" : [ "full_hostname", "domain_without_subdomains" ], + * "config" : { + * "full_hostname" : "URL_TO_HOST(url)", + * "domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + * } + * } + * ] + * } + */ + @Multiline + public static String squidParserConfig; + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "squid", + * "parserConfig": { + * "grokPath": "/patterns/squid", + * "patternLabel": "SQUID_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR", + * "output" : [ "full_hostname", "domain_without_subdomains" ], + * "config" : { + * "full_hostname" : "URL_TO_HOST(url)", + * "domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + * } + * } + */ + @Multiline + public static String badParserConfig; + + + /** + * { + * "enrichment" : { + * "fieldMap": + * { + * "geo": ["ip_dst_addr", "ip_src_addr"], + * "host": ["host"] + * } + * }, + * "threatIntel" : { + * "fieldMap": + * { + * "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] + * }, + * "fieldToTypeMap": + * { + * "ip_src_addr" : ["malicious_ip"], + * "ip_dst_addr" : ["malicious_ip"] + * }, + * "triageConfig" : { + * "riskLevelRules" : [ + * { + * "rule" : "not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))", + * "score" : 10 + * } + * ], + * "aggregator" : "MAX" + * } + * } + * } + */ + @Multiline + public static String snortEnrichmentConfig; + + /** + * { + * "enrichment" : { + * "fieldMap": + * { + * "geo": ["ip_dst_addr", "ip_src_addr"], + * "host": ["host"] + * } + * }, + * "threatIntel" : { + * "fieldMap": + * { + * "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] + * }, + * "fieldToTypeMap": + * { + * "ip_src_addr" : ["malicious_ip"], + * "ip_dst_addr" : ["malicious_ip"] + * }, + * "triageConfig" : { + * "riskLevelRules" : [ + * { + * "rule" : "not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24')))", + * "score" : 10 + * } + * ], + * "aggregator" : "MAX" + * } + * } + * } + */ + @Multiline + public static String badEnrichmentConfig; + + /** + * { + * "profiles": [ + * { + * "profile": "example2", + * "foreach": "ip_src_addr", + * "onlyif": "protocol == 'DNS' or protocol == 'HTTP'", + * "init": { + * "num_dns": 1.0, + * "num_http": 1.0 + * }, + * "update": { + * "num_dns": "num_dns + (if protocol == 'DNS' then 1 else 0)", + * "num_http": "num_http + (if protocol == 'HTTP' then 1 else 0)" + * }, + * "result": "num_dns / num_http" + * }, + * { + * "profile": "example1", + * "foreach": "ip_src_addr", + * "onlyif": "protocol == 'HTTP'", + * "init": { + * "total_bytes": 0.0 + * }, + * "update": { + * "total_bytes": "total_bytes + bytes_in" + * }, + * "result": "total_bytes", + * "expires": 30 + * }, + * { + * "profile": "percentiles", + * "foreach": "ip_src_addr", + * "onlyif": "protocol == 'HTTP'", + * "init": { "s": "STATS_INIT(100)" }, + * "update": { "s": "STATS_ADD(s, length)" }, + * "result": "STATS_PERCENTILE(s, 0.7)" + * } + * ] + * } + */ + @Multiline + public static String profilerConfig; + + @Test + public void testVistVisitsAllConfiguredNodes() throws Exception { + + ConfigurationsUtils + .writeSensorParserConfigToZookeeper("squid", squidParserConfig.getBytes(), zookeeperUrl); + + ConfigurationsUtils + .writeSensorEnrichmentConfigToZookeeper("snort", snortEnrichmentConfig.getBytes(), + zookeeperUrl); + ConfigurationsUtils.writeProfilerConfigToZookeeper(profilerConfig.getBytes(), client); + + StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); + stellarStatementReporter.vist(client, (names, statement) -> { + String name = String.join("->", names); + Assert.assertTrue(expected.contains(name)); + expected.remove(name); + }, (names, error) -> Assert.assertTrue("Should not get errors", false)); + + Assert.assertTrue(expected.isEmpty()); + } + + @Test + public void testErrorCalledWithBadParserConfig() throws Exception { + // calling this way to avoid getting the serialize error in ConfigurationUtils instead of visit + ConfigurationsUtils.writeToZookeeper(ConfigurationType.PARSER.getZookeeperRoot() + "/" + "squid", badParserConfig.getBytes(),client); + StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); + stellarStatementReporter.vist(client, (names, statement) -> { + Assert.assertTrue("This should have failed", false); + }, (names, error) -> { + Assert.assertTrue("Should get errors", true); + }); + } + + @Test(expected = RuntimeException.class) + public void testExceptionThrownGetsBadThreatConfig() throws Exception{ + ConfigurationsUtils + .writeSensorEnrichmentConfigToZookeeper("snort", badEnrichmentConfig.getBytes(), + zookeeperUrl); + StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); + stellarStatementReporter.vist(client, (names, statement) -> { + Assert.assertTrue("This should have failed", false); + }, (names, error) -> Assert.assertTrue("Should not get errors", false)); + } +} \ No newline at end of file From c0315b8291557de94dcf701d8def16d0b2866798 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Thu, 30 Nov 2017 20:18:45 -0500 Subject: [PATCH 04/14] refactor to utility classes, first step in major refactor --- .../stellar/common/shell/StellarShell.java | 65 +---------- .../utils/validation/StellarValidator.java | 28 +++++ .../StellarZookeeperBasedValidator.java | 110 ++++++++++++++++++ 3 files changed, 141 insertions(+), 62 deletions(-) create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index 84a1d4d353..dfcfde8307 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -36,6 +36,7 @@ import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.common.configuration.ConfigurationsUtils; import org.apache.metron.stellar.common.utils.JSONUtils; +import org.apache.metron.stellar.common.utils.validation.StellarZookeeperBasedValidator; import org.apache.metron.stellar.dsl.StellarFunctionInfo; import org.atteo.classindex.ClassIndex; import org.jboss.aesh.complete.CompleteOperation; @@ -427,68 +428,8 @@ private void handleValidateConfigured() { return; } - // discover all the StellarConfiguredStatementReporters - Set reporterSet = new HashSet<>(); - - for (Class c : ClassIndex.getSubclasses(StellarConfiguredStatementReporter.class, - Thread.currentThread().getContextClassLoader())) { - boolean isAssignable = StellarConfiguredStatementReporter.class.isAssignableFrom(c); - if (isAssignable) { - try { - StellarConfiguredStatementReporter reporter = StellarConfiguredStatementReporter.class - .cast(c.getConstructor().newInstance()); - reporterSet.add(reporter); - } catch (Exception e) { - writeLine(ERROR_PROMPT + " Reporter: " + c.getCanonicalName() + " not valid, skipping"); - } - } - } - - writeLine(String.format("Discovered %d reporters", reporterSet.size())); - - writeLine("Visiting all configurations. ThreatTriage rules are checked when loading the " - + "configuration, thus an invalid ThreatTriage rule will fail the entire Enrichement Configuration."); - if (reporterSet.size() > 0) { - reporterSet.forEach((r) -> write(r.getName() + " ")); - writeLine(""); - } else { - return; - } - - reporterSet.forEach((r) -> { - writeLine("Visiting " + r.getName()); - try { - r.vist(client.get(), ((contextNames, statement) -> { - // the names taken together are the identifier for this - // statement - String name = String.join("->", contextNames); - - writeLine("=================================================="); - writeLine("validating " + name); - try { - if (StellarProcessor.compile(statement) == null) { - writeLine( - ERROR_PROMPT + String.format("Statement: %s is not valid, please review", name)); - writeLine(ERROR_PROMPT + String.format("Statement: %s ", statement)); - } - } catch (RuntimeException e) { - writeLine(ERROR_PROMPT + "Error Visiting " + name); - writeLine(e.getMessage()); - writeLine("--"); - writeLine(ERROR_PROMPT + ": " + statement); - } - writeLine("=================================================="); - }), (contextNames, exception) -> { - String name = String.join("->", contextNames); - writeLine( - ERROR_PROMPT + String.format("Configuration %s is not valid, please review", name)); - }); - } catch (Exception e) { - writeLine(ERROR_PROMPT + "Error Visiting " + r.getName()); - writeLine(e.getMessage()); - } - }); - writeLine("\nDone validation"); + StellarZookeeperBasedValidator validator = new StellarZookeeperBasedValidator(client.get()); + validator.validate(this::writeLine); } /** diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java new file mode 100644 index 0000000000..16bcda0076 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java @@ -0,0 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation; + +public interface StellarValidator { + public interface LineWriter { + void write(String line); + } + void validate(LineWriter writer); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java new file mode 100644 index 0000000000..96343ab713 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java @@ -0,0 +1,110 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation; + +import static org.apache.metron.stellar.common.shell.StellarShell.ERROR_PROMPT; + +import java.util.HashSet; +import java.util.Set; +import org.apache.commons.lang.NullArgumentException; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.StellarConfiguredStatementReporter; +import org.apache.metron.stellar.common.StellarProcessor; +import org.atteo.classindex.ClassIndex; + +public class StellarZookeeperBasedValidator implements StellarValidator{ + + private CuratorFramework client; + + public StellarZookeeperBasedValidator(CuratorFramework client) throws NullArgumentException { + if(client == null) { + throw new NullArgumentException("client"); + } + this.client = client; + } + + + @Override + public void validate(LineWriter writer) { + // discover all the StellarConfiguredStatementReporters + Set reporterSet = new HashSet<>(); + + for (Class c : ClassIndex.getSubclasses(StellarConfiguredStatementReporter.class, + Thread.currentThread().getContextClassLoader())) { + boolean isAssignable = StellarConfiguredStatementReporter.class.isAssignableFrom(c); + if (isAssignable) { + try { + StellarConfiguredStatementReporter reporter = StellarConfiguredStatementReporter.class + .cast(c.getConstructor().newInstance()); + reporterSet.add(reporter); + } catch (Exception e) { + writer.write(ERROR_PROMPT + " Reporter: " + c.getCanonicalName() + " not valid, skipping"); + } + } + } + + writer.write(String.format("Discovered %d reporters", reporterSet.size())); + + writer.write("Visiting all configurations. ThreatTriage rules are checked when loading the " + + "configuration, thus an invalid ThreatTriage rule will fail the entire Enrichement Configuration."); + if (reporterSet.size() > 0) { + reporterSet.forEach((r) -> writer.write(r.getName() + " ")); + writer.write(""); + } else { + return; + } + + reporterSet.forEach((r) -> { + writer.write("Visiting " + r.getName()); + try { + r.vist(client, ((contextNames, statement) -> { + // the names taken together are the identifier for this + // statement + String name = String.join("->", contextNames); + + writer.write("=================================================="); + writer.write("validating " + name); + try { + if (StellarProcessor.compile(statement) == null) { + writer.write( + ERROR_PROMPT + String.format("Statement: %s is not valid, please review", name)); + writer.write(ERROR_PROMPT + String.format("Statement: %s ", statement)); + } + } catch (RuntimeException e) { + writer.write(ERROR_PROMPT + "Error Visiting " + name); + writer.write(e.getMessage()); + writer.write("--"); + writer.write(ERROR_PROMPT + ": " + statement); + } + writer.write("=================================================="); + }), (contextNames, exception) -> { + String name = String.join("->", contextNames); + writer.write( + ERROR_PROMPT + String.format("Configuration %s is not valid, please review", name)); + }); + } catch (Exception e) { + writer.write(ERROR_PROMPT + "Error Visiting " + r.getName()); + writer.write(e.getMessage()); + } + }); + writer.write("\nDone validation"); + } +} From 65278a67a07f1c4c23ab2d95ebb6de92e1cac731 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Fri, 1 Dec 2017 11:46:57 -0500 Subject: [PATCH 05/14] Refactor based on review and inspiration from review. Although the original implementation was functional, it required maintainence to keep current. The suggested 'best state' was to have it be possible, maybe through annotations, for the validation system to be able to handle any config, regarless or composition using annotations. That would leave it up to the implementor to propertly annotate thier configurations, and allow for support of new fields. This is an implementation of that. I have refactored the implemenations and details, but kept the discovery and mechanics ( loading and visitation ) somewhat the same. Hopefully keeping the good and reworking to a more sustainable solution. Several annotations where created to marks ceratin stellar configruation objects or scenarios. A holder object, to hold the configuration object, but knows how to process the annotations and run the visitation was added. This holder object and the annotations have parameters and handling for several special scenarios, such as 2x nested maps. This implementation should facilitate follow on work to validate files and streams and blobs by using implementing the StellarValidator interface and re-using the holder concept ( replacing the providers ) --- dependencies_with_url.csv | 2 + .../configuration/FieldTransformer.java | 6 + .../configuration/SensorParserConfig.java | 4 + .../StellarStatementReporter.java | 210 +++------------ .../enrichment/EnrichmentConfig.java | 4 + .../enrichment/SensorEnrichmentConfig.java | 4 + .../enrichment/threatintel/RiskLevelRule.java | 6 + .../threatintel/ThreatIntelConfig.java | 3 + .../threatintel/ThreatTriageConfig.java | 2 + .../configuration/profiler/ProfileConfig.java | 10 + .../configuration/profiler/ProfileResult.java | 4 + .../profiler/ProfileResultExpressions.java | 4 + .../profiler/ProfileTriageExpressions.java | 4 + .../profiler/ProfilerConfig.java | 4 + .../transformation/StellarTransformation.java | 1 + .../StellarStatementReporterTest.java | 101 ++++--- metron-stellar/stellar-common/pom.xml | 2 +- .../stellar/common/shell/StellarShell.java | 8 - .../ExpressionConfigurationHolder.java | 253 ++++++++++++++++++ .../StellarConfigurationProvider.java} | 26 +- .../StellarConfiguredStatementVisitor.java | 37 +++ .../StellarZookeeperBasedValidator.java | 85 +++--- .../annotations/StellarConfiguration.java | 28 ++ .../annotations/StellarConfigurationList.java | 29 ++ .../annotations/StellarExpressionField.java | 29 ++ .../annotations/StellarExpressionList.java | 30 +++ .../annotations/StellarExpressionMap.java | 62 +++++ 27 files changed, 693 insertions(+), 265 deletions(-) create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java rename metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/{StellarConfiguredStatementReporter.java => utils/validation/StellarConfigurationProvider.java} (60%) create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementVisitor.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 38a9f5e70e..4507e20839 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -174,6 +174,8 @@ commons-lang:commons-lang:jar:2.4:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.5:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.6:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.6:provided,ASLv2,http://commons.apache.org/lang/ +commons-lang:commons-lang:jar:3.7:compile,ASLv2,http://commons.apache.org/lang/ +commons-lang:commons-lang:jar:3.7:provided,ASLv2,http://commons.apache.org/lang/ commons-logging:commons-logging:jar:1.1.1:compile,ASLv2,http://commons.apache.org/logging commons-logging:commons-logging:jar:1.1.3:compile,ASLv2,http://commons.apache.org/proper/commons-logging/ commons-logging:commons-logging:jar:1.2:compile,ASLv2,http://commons.apache.org/proper/commons-logging/ diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java index df80691ddd..9c76237506 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java @@ -20,6 +20,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableList; +import org.apache.metron.common.field.transformation.StellarTransformation; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.common.field.transformation.FieldTransformation; import org.apache.metron.common.field.transformation.FieldTransformations; @@ -28,11 +31,14 @@ import java.io.Serializable; import java.util.*; +@StellarConfigurationList(name = "FieldTransformer") public class FieldTransformer implements Serializable { private List input = new ArrayList<>(); private List output; private FieldTransformation transformation; private String transformationName; + @StellarExpressionMap(name = "Field Mapping", + qualify_with_field = "transformation", qualify_with_field_type = StellarTransformation.class) private LinkedHashMap config = new LinkedHashMap<>(); private boolean initialized = false; public FieldTransformer() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java index 2d0ccd8027..af4bfd7a51 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java @@ -26,7 +26,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; +@StellarConfiguration public class SensorParserConfig implements Serializable { private String parserClassName; @@ -229,6 +232,7 @@ public void setWriterClassName(String classNames) { this.writerClassName = classNames; } private Map parserConfig = new HashMap<>(); + @StellarConfigurationList(name = "fieldTransformations") private List fieldTransformations = new ArrayList<>(); public List getFieldTransformations() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java index fcb745ad5d..4491234229 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java @@ -20,14 +20,11 @@ import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; import java.util.Arrays; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; -import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.configuration.profiler.ProfileResult; @@ -36,18 +33,16 @@ import org.apache.metron.common.configuration.profiler.ProfilerConfig; import org.apache.metron.common.field.transformation.StellarTransformation; import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.stellar.common.StellarConfiguredStatementReporter; +import org.apache.metron.stellar.common.utils.validation.ExpressionConfigurationHolder; +import org.apache.metron.stellar.common.utils.validation.StellarConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementVisitor.ErrorConsumer; import org.apache.zookeeper.KeeperException.NoNodeException; /** * StellarStatementReporter is used to report all of the configured / deployed Stellar statements in * the system. */ -public class StellarStatementReporter implements StellarConfiguredStatementReporter { - - public enum Type { - ENRICHMENT, THREAT_INTEL - } +public class StellarStatementReporter implements StellarConfigurationProvider { public StellarStatementReporter() { } @@ -58,58 +53,53 @@ public String getName() { } @Override - public void vist(CuratorFramework client, StatementReportVisitor visitor, - ConfigReportErrorConsumer errorConsumer) throws Exception { - visitParserConfigs(client, visitor, errorConsumer); - visitEnrichmentConfigs(client, visitor, errorConsumer); - visitProfilerConfigs(client, visitor, errorConsumer); + public List provideConfigurations(CuratorFramework client, + ErrorConsumer errorConsumer) { + List holders = new LinkedList<>(); + visitParserConfigs(client, holders, errorConsumer); + visitEnrichmentConfigs(client, holders, errorConsumer); + visitProfilerConfigs(client, holders, errorConsumer); + return holders; } - private void visitParserConfigs(CuratorFramework client, StatementReportVisitor visitor, - ConfigReportErrorConsumer errorConsumer) throws Exception { + private void visitParserConfigs(CuratorFramework client, + List holders, ErrorConsumer errorConsumer) { List children = null; try { children = client.getChildren().forPath(PARSER.getZookeeperRoot()); - } catch (NoNodeException nne) { + } catch (Exception nne) { return; } - for (String child : children) { - byte[] data = client.getData().forPath(PARSER.getZookeeperRoot() + "/" + child); try { + byte[] data = client.getData().forPath(PARSER.getZookeeperRoot() + "/" + child); SensorParserConfig parserConfig = SensorParserConfig.fromBytes(data); - List transformations = parserConfig.getFieldTransformations(); - transformations.forEach((f) -> { - if (StellarTransformation.class.isAssignableFrom(f.getFieldTransformation().getClass())) { - f.getConfig().forEach((k, v) -> { - List names = Arrays - .asList(getName(), PARSER.toString(), parserConfig.getSensorTopic(), k); - visitor.visit(names, v.toString()); - }); - } - }); + ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder( + String.format("%s/%s", getName(), PARSER.toString()), parserConfig.getSensorTopic(), + parserConfig); + holders.add(holder); } catch (Exception e) { - List names = Arrays.asList(getName(), PARSER.toString(), child); - errorConsumer.consume(names, e); + errorConsumer.consume(String.format("%s/%s/%s", getName(), PARSER.toString(), child), e); } } } @SuppressWarnings("unchecked") - private void visitEnrichmentConfigs(CuratorFramework client, StatementReportVisitor visitor, - ConfigReportErrorConsumer errorConsumer) throws Exception { + private void visitEnrichmentConfigs(CuratorFramework client, + List holders, ErrorConsumer errorConsumer) { List children = null; try { children = client.getChildren().forPath(ENRICHMENT.getZookeeperRoot()); - } catch (NoNodeException nne) { + } catch (Exception nne) { return; } for (String child : children) { - byte[] data = client.getData().forPath(ENRICHMENT.getZookeeperRoot() + "/" + child); + try { + byte[] data = client.getData().forPath(ENRICHMENT.getZookeeperRoot() + "/" + child); // Certain parts of the SensorEnrichmentConfig do Stellar Verification on their // own as part of deserialization, where the bean spec will call the setter, which has // been wired with stellar verification calls. @@ -123,7 +113,11 @@ private void visitEnrichmentConfigs(CuratorFramework client, StatementReportVisi // though they can be executed, then they will never fail. final SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig .fromBytes(data); - + ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder( + String.format("%s/%s", getName(), ENRICHMENT.toString()), child, + sensorEnrichmentConfig); + holders.add(holder); + /* EnrichmentConfig enrichmentConfig; enrichmentConfig = sensorEnrichmentConfig.getEnrichment(); visitEnrichmentConfig(child, Type.ENRICHMENT, enrichmentConfig, visitor, errorConsumer); @@ -132,77 +126,16 @@ private void visitEnrichmentConfigs(CuratorFramework client, StatementReportVisi final ThreatTriageConfig threatTriageConfig = sensorEnrichmentConfig.getThreatIntel() .getTriageConfig(); visitEnrichmentThreatTriageConfigs(child, threatTriageConfig, visitor, errorConsumer); + */ } catch (Exception e) { - List names = Arrays.asList(getName(), ENRICHMENT.toString(), child); - errorConsumer.consume(names, e); + errorConsumer + .consume(String.format("%s/%s/%s", getName(), ENRICHMENT.toString(), child), e); } } } - @SuppressWarnings("unchecked") - private void visitEnrichmentConfig(String topicName, Type type, - EnrichmentConfig enrichmentConfig, StatementReportVisitor visitor, - ConfigReportErrorConsumer errorConsumer) { - - Map enrichmentStellarMap = (Map) enrichmentConfig.getFieldMap() - .getOrDefault("stellar", new HashMap<>()); - Map transforms = (Map) enrichmentStellarMap - .getOrDefault("config", new HashMap<>()); - try { - for (Map.Entry kv : transforms.entrySet()) { - // we can have a group or an entry - if (kv.getValue() instanceof Map) { - Map groupMap = (Map) kv.getValue(); - for (Map.Entry groupKv : groupMap.entrySet()) { - List names = Arrays - .asList(getName(), ENRICHMENT.toString(), topicName, type.toString(), kv.getKey(), - groupKv.getKey()); - visitor.visit(names, groupKv.getValue()); - } - } else { - List names = Arrays - .asList(getName(), ENRICHMENT.toString(), topicName, type.toString(), "(default)", - kv.getKey(), kv.getKey()); - visitor.visit(names, kv.getValue().toString()); - } - } - } catch (Exception e) { - List names = Arrays - .asList(getName(), ENRICHMENT.toString(), topicName, type.toString()); - errorConsumer.consume(names, e); - } - } - - @SuppressWarnings("unchecked") - private void visitEnrichmentThreatTriageConfigs(String topicName, - ThreatTriageConfig threatTriageConfig, StatementReportVisitor visitor, - ConfigReportErrorConsumer errorConsumer) { - try { - List riskLevelRules = threatTriageConfig.getRiskLevelRules(); - riskLevelRules.forEach((r) -> { - String name = r.getName(); - if (org.apache.commons.lang.StringUtils.isEmpty(name)) { - name = "(default)"; - } - List names = Arrays - .asList(getName(), ENRICHMENT.toString(), topicName, "THREAT_TRIAGE", name, "rule"); - visitor.visit(names, r.getRule()); - if (!org.apache.commons.lang.StringUtils.isEmpty(r.getReason())) { - names = Arrays - .asList(getName(), ENRICHMENT.toString(), topicName, "THREAT_TRIAGE", name, "reason"); - visitor.visit(names, r.getReason()); - } - }); - - } catch (Exception e) { - List names = Arrays - .asList(getName(), ENRICHMENT.toString(), topicName, "THREAT_TRIAGE"); - errorConsumer.consume(names, e); - } - } - - private void visitProfilerConfigs(CuratorFramework client, StatementReportVisitor visitor, - ConfigReportErrorConsumer errorConsumer) { + private void visitProfilerConfigs(CuratorFramework client, + List holders, ErrorConsumer errorConsumer) { try { byte[] profilerConfigData = null; try { @@ -214,77 +147,12 @@ private void visitProfilerConfigs(CuratorFramework client, StatementReportVisito ProfilerConfig profilerConfig = JSONUtils.INSTANCE .load(new String(profilerConfigData), ProfilerConfig.class); profilerConfig.getProfiles().forEach((ProfileConfig pc) -> { - List names = new LinkedList<>(); - names.add(getName()); - names.add(PROFILER.toString()); - names.add(pc.getProfile()); - - // only if - if (!org.apache.commons.lang.StringUtils.isEmpty(pc.getOnlyif())) { - names.add("only_if"); - visitor.visit(names, pc.getOnlyif()); - names.remove("only_if"); - } - - // init - if (!pc.getInit().isEmpty()) { - names.add("init"); - pc.getInit().forEach((k, v) -> { - names.add(k); - visitor.visit(names, v); - names.remove(k); - }); - names.remove("init"); - } - - // update - if (!pc.getUpdate().isEmpty()) { - names.add("update"); - pc.getUpdate().forEach((k, v) -> { - names.add(k); - visitor.visit(names, v); - names.remove(k); - }); - names.remove("update"); - } - - // group by - if (!pc.getGroupBy().isEmpty()) { - names.add("group_by"); - pc.getGroupBy().forEach((gb) -> visitor.visit(names, gb)); - names.remove("group_by"); - } - - // profile result - if (pc.getResult() != null) { - ProfileResult result = pc.getResult(); - ProfileResultExpressions profileResultExpressions = result.getProfileExpressions(); - ProfileTriageExpressions profileTriageExpressions = result.getTriageExpressions(); - - names.add("profile_result"); - if (profileResultExpressions != null && !org.apache.commons.lang.StringUtils - .isEmpty(profileResultExpressions.getExpression())) { - names.add("profile_expression"); - visitor.visit(names, profileResultExpressions.getExpression()); - names.remove("profile_expression"); - } - - if (profileTriageExpressions != null && !profileTriageExpressions.getExpressions() - .isEmpty()) { - names.add("triage_expression"); - profileTriageExpressions.getExpressions().forEach((k, v) -> { - names.add(k); - visitor.visit(names, v); - names.remove(k); - }); - } - } - - + ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder( + String.format("%s/%s", getName(), PROFILER.toString()), pc.getProfile(), pc); + holders.add(holder); }); } catch (Exception e) { - List names = Arrays.asList(getName(), PROFILER.toString()); - errorConsumer.consume(names, e); + errorConsumer.consume(String.format("%s/%s", getName(), PROFILER.toString()), e); } } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java index 3261c9f19f..c7bab0ba37 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java @@ -24,8 +24,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; +@StellarConfiguration public class EnrichmentConfig { + @StellarExpressionMap(inner_map_keys = {"stellar","config"}) private Map fieldMap = new HashMap<>(); private Map enrichmentConfigs = new HashMap<>(); private Map> fieldToTypeMap = new HashMap<>(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java index 2b4f5a89c5..3f94ed4d11 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java @@ -24,10 +24,14 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +@StellarConfiguration public class SensorEnrichmentConfig { + @StellarConfiguration private EnrichmentConfig enrichment = new EnrichmentConfig(); + @StellarConfiguration private ThreatIntelConfig threatIntel = new ThreatIntelConfig(); private Map configuration = new HashMap<>(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java index 94ab0c8d63..d9f93ae644 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java @@ -17,6 +17,9 @@ */ package org.apache.metron.common.configuration.enrichment.threatintel; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; + /** * This class represents a rule that is used to triage threats. * @@ -29,6 +32,7 @@ * Tuning the threat triage process involves creating one or more rules, adjusting * the score of each rule, and changing the way that each rule's score is aggregated. */ +@StellarConfiguration public class RiskLevelRule { /** @@ -45,6 +49,7 @@ public class RiskLevelRule { * A predicate, in the form of a Stellar expression, that determines whether * the rule is applied to an alert or not. This field is required. */ + @StellarExpressionField(name = "rule") String rule; /** @@ -60,6 +65,7 @@ public class RiskLevelRule { * This is expected to be a valid Stellar expression and can refer to any of the * fields within the message itself. */ + @StellarExpressionField(name = "reason") String reason; public String getName() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java index 3b34a0af37..a3721e0955 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java @@ -19,8 +19,11 @@ package org.apache.metron.common.configuration.enrichment.threatintel; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +@StellarConfiguration public class ThreatIntelConfig extends EnrichmentConfig { + @StellarConfiguration private ThreatTriageConfig triageConfig = new ThreatTriageConfig(); public ThreatTriageConfig getTriageConfig() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java index 961f9d56ea..81d386559d 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java @@ -29,9 +29,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; public class ThreatTriageConfig { + @StellarConfigurationList(name = "riskLevelRules") private List riskLevelRules = new ArrayList<>(); private Aggregators aggregator = Aggregators.MAX; private Map aggregationConfig = new HashMap<>(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java index 06c82d26c7..5dd3e51166 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java @@ -26,10 +26,15 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; /** * The definition of a single Profile. */ +@StellarConfiguration public class ProfileConfig implements Serializable { /** @@ -52,6 +57,7 @@ public class ProfileConfig implements Serializable { * is only applied to a profile if this condition is true. This allows a profile * to filter the messages that it receives. */ + @StellarExpressionField(name = "onlyif") private String onlyif = "true"; /** @@ -61,6 +67,7 @@ public class ProfileConfig implements Serializable { * period the expression is executed once and stored in a variable with the given * name. */ + @StellarExpressionMap(name = "init") private Map init = new HashMap<>(); /** @@ -68,12 +75,14 @@ public class ProfileConfig implements Serializable { * A map is expected where the key is the variable name and the value is a Stellar * expression. The map can include 0 or more variables/expressions. */ + @StellarExpressionMap(name = "update") private Map update = new HashMap<>(); /** * A list of Stellar expressions that is executed in order and used to group the * resulting profile data. */ + @StellarExpressionList(name = "groupBy") private List groupBy = new ArrayList<>(); /** @@ -81,6 +90,7 @@ public class ProfileConfig implements Serializable { * expression(s) are expected to in some way summarize the messages that were applied * to the profile over the window period. */ + @StellarConfiguration private ProfileResult result; /** diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java index 55642a938a..f12fb1c3da 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java @@ -19,10 +19,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; /** * Defines the 'result' field of a Profile definition. */ +@StellarConfiguration public class ProfileResult { /** @@ -30,6 +32,7 @@ public class ProfileResult { * a measurement that is persisted in the profile store. */ @JsonProperty("profile") + @StellarConfiguration private ProfileResultExpressions profileExpressions; /** @@ -38,6 +41,7 @@ public class ProfileResult { * triage. */ @JsonProperty("triage") + @StellarConfiguration private ProfileTriageExpressions triageExpressions; @JsonCreator diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java index 82af223d86..57ba7a1275 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java @@ -19,14 +19,18 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; /** * A Stellar expression that is executed to produce a single * measurement that is persisted within the profile store. */ +@StellarConfiguration public class ProfileResultExpressions { @JsonIgnore + @StellarExpressionField(name = "expression") private String expression; @JsonCreator diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java index fbe1706b4d..a1a57d26f8 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java @@ -22,6 +22,8 @@ import java.util.HashMap; import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; /** * A set of Stellar expressions that are executed to produce a single @@ -30,6 +32,7 @@ * The result of evaluating each expression are made available, keyed * by the given name, to the threat triage process. */ +@StellarConfiguration public class ProfileTriageExpressions { /** @@ -40,6 +43,7 @@ public class ProfileTriageExpressions { * or map of basic data types that can be serialized. */ @JsonIgnore + @StellarExpressionMap(name = "expressions") private Map expressions; @JsonCreator diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java index e7c081a0a2..07639250ef 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java @@ -20,15 +20,19 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; /** * The definition for entire Profiler, which may contain many Profile definitions. */ +@StellarConfiguration public class ProfilerConfig implements Serializable { /** * One or more profile definitions. */ + @StellarConfigurationList(name = "profiles") private List profiles = new ArrayList<>(); public List getProfiles() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java index 2a22e2168b..b9c97f5329 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java @@ -18,6 +18,7 @@ package org.apache.metron.common.field.transformation; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java index 36deaa3793..84f72058c5 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java @@ -23,6 +23,7 @@ import org.adrianwalker.multilinestring.Multiline; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.test.TestingServer; +import org.apache.metron.stellar.common.utils.validation.ExpressionConfigurationHolder; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -34,30 +35,43 @@ public class StellarStatementReporterTest { private String zookeeperUrl; private CuratorFramework client; private List expected; - + private List expectedStatements; + private List foundStatements; @Before public void setUp() throws Exception { testZkServer = new TestingServer(true); zookeeperUrl = testZkServer.getConnectString(); client = ConfigurationsUtils.getClient(zookeeperUrl); expected = new LinkedList<>(); - expected.add("Apache Metron->PARSER->squid->full_hostname"); - expected.add("Apache Metron->PARSER->squid->domain_without_subdomains"); - expected.add("Apache Metron->ENRICHMENT->snort->THREAT_TRIAGE->(default)->rule"); - expected.add("Apache Metron->PROFILER->example2->only_if"); - expected.add("Apache Metron->PROFILER->example2->init->num_dns"); - expected.add("Apache Metron->PROFILER->example2->init->num_http"); - expected.add("Apache Metron->PROFILER->example2->update->num_dns"); - expected.add("Apache Metron->PROFILER->example2->update->num_http"); - expected.add("Apache Metron->PROFILER->example2->profile_result->profile_expression"); - expected.add("Apache Metron->PROFILER->example1->only_if"); - expected.add("Apache Metron->PROFILER->example1->init->total_bytes"); - expected.add("Apache Metron->PROFILER->example1->update->total_bytes"); - expected.add("Apache Metron->PROFILER->example1->profile_result->profile_expression"); - expected.add("Apache Metron->PROFILER->percentiles->only_if"); - expected.add("Apache Metron->PROFILER->percentiles->init->s"); - expected.add("Apache Metron->PROFILER->percentiles->update->s"); - expected.add("Apache Metron->PROFILER->percentiles->profile_result->profile_expression"); + expected.add("Apache Metron/PARSER/squid"); + expected.add("Apache Metron/ENRICHMENT/snort"); + expected.add("Apache Metron/PROFILER/example2"); + expected.add("Apache Metron/PROFILER/example1"); + expected.add("Apache Metron/PROFILER/percentiles"); + + expectedStatements = new LinkedList<>(); + expectedStatements.add("Apache Metron/PARSER/squid/fieldTransformations/0/Field Mapping/full_hostname"); + expectedStatements.add("Apache Metron/PARSER/squid/fieldTransformations/0/Field Mapping/domain_without_subdomains"); + expectedStatements.add("Apache Metron/ENRICHMENT/snort/enrichment/default/foo"); + expectedStatements.add("Apache Metron/ENRICHMENT/snort/enrichment/default/ALL_CAPS"); + expectedStatements.add("Apache Metron/ENRICHMENT/snort/threatIntel/triageConfig/riskLevelRules/0/rule"); + expectedStatements.add("Apache Metron/PROFILER/example2/onlyif"); + expectedStatements.add("Apache Metron/PROFILER/example2/init/num_dns"); + expectedStatements.add("Apache Metron/PROFILER/example2/init/num_http"); + expectedStatements.add("Apache Metron/PROFILER/example2/update/num_dns"); + expectedStatements.add("Apache Metron/PROFILER/example2/update/num_http"); + expectedStatements.add("Apache Metron/PROFILER/example2/result/profileExpressions/expression"); + expectedStatements.add("Apache Metron/PROFILER/example1/onlyif"); + expectedStatements.add("Apache Metron/PROFILER/example1/init/total_bytes"); + expectedStatements.add("Apache Metron/PROFILER/example1/update/total_bytes"); + expectedStatements.add("Apache Metron/PROFILER/example1/result/profileExpressions/expression"); + expectedStatements.add("Apache Metron/PROFILER/percentiles/onlyif"); + expectedStatements.add("Apache Metron/PROFILER/percentiles/init/s"); + expectedStatements.add("Apache Metron/PROFILER/percentiles/update/s"); + expectedStatements.add("Apache Metron/PROFILER/percentiles/result/profileExpressions/expression"); + + foundStatements = new LinkedList<>(); + client.start(); } @@ -122,7 +136,14 @@ public void tearDown() throws Exception { * "fieldMap": * { * "geo": ["ip_dst_addr", "ip_src_addr"], - * "host": ["host"] + * "host": ["host"], + * "stellar" : { + * "type" : "STELLAR", + * "config" : { + * "foo" : "1 + 1", + * "ALL_CAPS" : "TO_UPPER(source.type)" + * } + * } * } * }, * "threatIntel" : { @@ -229,8 +250,7 @@ public void tearDown() throws Exception { public static String profilerConfig; @Test - public void testVistVisitsAllConfiguredNodes() throws Exception { - + public void testProvideAllConfiguredNodes() throws Exception { ConfigurationsUtils .writeSensorParserConfigToZookeeper("squid", squidParserConfig.getBytes(), zookeeperUrl); @@ -240,35 +260,48 @@ public void testVistVisitsAllConfiguredNodes() throws Exception { ConfigurationsUtils.writeProfilerConfigToZookeeper(profilerConfig.getBytes(), client); StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); - stellarStatementReporter.vist(client, (names, statement) -> { - String name = String.join("->", names); - Assert.assertTrue(expected.contains(name)); - expected.remove(name); - }, (names, error) -> Assert.assertTrue("Should not get errors", false)); + List holders = stellarStatementReporter + .provideConfigurations(client, + (names, error) -> Assert.assertTrue("Should not get errors", false)); + holders.forEach((h) -> { + try { + h.discover(); + h.visit((p,s) -> { + foundStatements.add(p); + },(s,e) -> { + Assert.assertTrue(e.getMessage(),false); + }); + } catch (IllegalAccessException e) { + Assert.assertTrue(e.getMessage(), false); + } + }); + holders.forEach((h) -> expected.remove(h.getFullName())); Assert.assertTrue(expected.isEmpty()); + + foundStatements.removeAll(expectedStatements); + Assert.assertTrue(foundStatements.isEmpty()); } @Test public void testErrorCalledWithBadParserConfig() throws Exception { // calling this way to avoid getting the serialize error in ConfigurationUtils instead of visit - ConfigurationsUtils.writeToZookeeper(ConfigurationType.PARSER.getZookeeperRoot() + "/" + "squid", badParserConfig.getBytes(),client); + ConfigurationsUtils + .writeToZookeeper(ConfigurationType.PARSER.getZookeeperRoot() + "/" + "squid", + badParserConfig.getBytes(), client); StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); - stellarStatementReporter.vist(client, (names, statement) -> { - Assert.assertTrue("This should have failed", false); - }, (names, error) -> { + stellarStatementReporter.provideConfigurations(client, (names, error) -> { Assert.assertTrue("Should get errors", true); }); } @Test(expected = RuntimeException.class) - public void testExceptionThrownGetsBadThreatConfig() throws Exception{ + public void testExceptionThrownGetsBadThreatConfig() throws Exception { ConfigurationsUtils .writeSensorEnrichmentConfigToZookeeper("snort", badEnrichmentConfig.getBytes(), zookeeperUrl); StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); - stellarStatementReporter.vist(client, (names, statement) -> { - Assert.assertTrue("This should have failed", false); - }, (names, error) -> Assert.assertTrue("Should not get errors", false)); + stellarStatementReporter.provideConfigurations(client, + (names, error) -> Assert.assertTrue("Should not get errors", false)); } } \ No newline at end of file diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml index 48df224538..dfd1fc16f2 100644 --- a/metron-stellar/stellar-common/pom.xml +++ b/metron-stellar/stellar-common/pom.xml @@ -64,7 +64,7 @@ org.apache.commons commons-lang3 - 3.2 + 3.7 commons-io diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index dfcfde8307..c1d73c3d9c 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -24,21 +24,14 @@ import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import java.io.PrintStream; -import java.util.HashSet; -import java.util.Set; import org.apache.commons.cli.*; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.log4j.PropertyConfigurator; import org.apache.metron.stellar.common.StellarAssignment; -import org.apache.metron.stellar.common.StellarConfiguredStatementReporter; -import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.configuration.ConfigurationsUtils; import org.apache.metron.stellar.common.utils.JSONUtils; import org.apache.metron.stellar.common.utils.validation.StellarZookeeperBasedValidator; import org.apache.metron.stellar.dsl.StellarFunctionInfo; -import org.atteo.classindex.ClassIndex; import org.jboss.aesh.complete.CompleteOperation; import org.jboss.aesh.complete.Completion; import org.jboss.aesh.console.AeshConsoleCallback; @@ -56,7 +49,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java new file mode 100644 index 0000000000..70ee72b2c3 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.lang.annotation.AnnotationTypeMismatchException; +import java.lang.annotation.IncompleteAnnotationException; +import java.lang.reflect.Field; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang.NullArgumentException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; + +public class ExpressionConfigurationHolder implements StellarConfiguredStatementVisitor { + + private Object holderObject; + private String name; + private String parentName; + private String fullName; + private List children = new LinkedList<>(); + private List expressionList; + private List expressionListList; + private List expresionMapList; + + public ExpressionConfigurationHolder(String parentName, String name, Object holderObject) { + if (holderObject == null) { + throw new NullArgumentException("holderObject"); + } + if (StringUtils.isEmpty(name)) { + throw new NullArgumentException("name"); + } + this.name = name; + this.parentName = parentName; + + this.fullName = StringUtils.isEmpty(parentName) ? this.name + : String.format("%s/%s", this.parentName, this.name); + + this.holderObject = holderObject; + } + + public Object getHolderObject() { + return holderObject; + } + + public String getName() { + return name; + } + + public String getParentName() { + return parentName; + } + + public String getFullName() { + return fullName; + } + + @Override + public void visit(StatementVisitor visitor, ErrorConsumer errorConsumer) { + visitExpressions(visitor, errorConsumer); + visitLists(visitor, errorConsumer); + visitMaps(visitor, errorConsumer); + visitChilden(visitor, errorConsumer); + } + + private void visitExpressions(StatementVisitor visitor, + ErrorConsumer errorConsumer) { + expressionList.forEach((f) -> { + String thisFullName = String + .format("%s/%s", getFullName(), f.getAnnotation(StellarExpressionField.class).name()); + try { + Object thisExpressionObject = FieldUtils.readField(f, holderObject, true); + if (thisExpressionObject == null || StringUtils.isEmpty(thisExpressionObject.toString())) { + return; + } + visitExpression(thisFullName,thisExpressionObject.toString(), visitor, + errorConsumer); + } catch (IllegalAccessException e) { + errorConsumer.consume(thisFullName, e); + } + }); + } + + private void visitExpression(String expressionName, String expression, + StatementVisitor visitor, ErrorConsumer errorConsumer) { + if (StringUtils.isEmpty(expression)) { + return; + } + try { + visitor.visit(expressionName, expression); + } catch (Exception e) { + errorConsumer.consume(expressionName, e); + } + } + + private void visitLists(StatementVisitor visitor, ErrorConsumer errorConsumer) { + expressionListList.forEach((l) -> { + String thisFullName = String + .format("%s/%s", getFullName(), l.getAnnotation(StellarExpressionList.class).name()); + try { + Object possibleIterable = FieldUtils.readField(l, holderObject, true); + if (possibleIterable == null) { + return; + } + Iterable it = (Iterable)possibleIterable; + int index = 0; + for (Object expressionObject : it) { + String expressionFullName = String.format("%s/%s", thisFullName, index); + visitExpression(expressionFullName, expressionObject.toString(), visitor, errorConsumer); + } + } catch (IllegalAccessException e) { + errorConsumer.consume(thisFullName, e); + } + }); + } + + @SuppressWarnings("unchecked") + private void visitMaps(StatementVisitor visitor, ErrorConsumer errorConsumer) { + expresionMapList.forEach((f) -> { + String thisFullName = String + .format("%s/%s", getFullName(), f.getAnnotation(StellarExpressionMap.class).name()); + try { + // if we have configured a qualifying field as a type of flag we need to check it + // for example, if StellarFoo.class.isAssignableFrom(this.foo.getClass()) then + // bar is a StellarExpressionMap + if (!StringUtils.isEmpty(f.getAnnotation(StellarExpressionMap.class).qualify_with_field())) { + String fieldName = f.getAnnotation(StellarExpressionMap.class).qualify_with_field(); + Class type = f.getAnnotation(StellarExpressionMap.class).qualify_with_field_type(); + Object theObject = FieldUtils.readField(holderObject, fieldName, true); + if (theObject == null) { + errorConsumer.consume(thisFullName, new IncompleteAnnotationException(StellarExpressionMap.class,"fieldName")); + return; + } + if (!type.isAssignableFrom(theObject.getClass())) { + return; + } + } + + Map map = (Map) FieldUtils.readField(f, holderObject, true); + + // some maps actually nest the config, so check and dig to get the real map + String[] innerKeys = f.getAnnotation(StellarExpressionMap.class).inner_map_keys(); + if (innerKeys.length != 0) { + for (String key : innerKeys) { + if (StringUtils.isEmpty(key)) { + return; + } + Object innerObject = map.get(key); + if (innerObject == null) { + return; + } + fullName = String.format("%s/%s",fullName,key); + if(!Map.class.isAssignableFrom(innerObject.getClass())) { + errorConsumer.consume(fullName, new Exception("The annotation specified an inner map that was not a map")); + } + map = (Map)innerObject; + } + } + + map.forEach((k, v) -> { + String mapKeyFullName = String.format("%s/%s", thisFullName, k.toString()); + if (Map.class.isAssignableFrom(v.getClass())) { + Map innerMap = (Map)v; + innerMap.forEach((ik,iv) -> { + if (iv == null) { + return; + } + visitExpression(String.format("%s/%s",mapKeyFullName,ik.toString()),iv.toString(), visitor, errorConsumer); + }); + return; + } + visitExpression(mapKeyFullName, v.toString(), visitor, errorConsumer); + }); + } catch (IllegalAccessException e) { + errorConsumer.consume(thisFullName, e); + } + }); + } + + private void visitChilden(StatementVisitor visitor, + ErrorConsumer errorConsumer) { + children.forEach((c) -> c.visit(visitor,errorConsumer)); + } + + public void discover() throws IllegalAccessException { + expressionList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionField.class); + expressionListList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionList.class); + expresionMapList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionMap.class); + List holderList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarConfiguration.class); + List holderListList = FieldUtils + .getFieldsListWithAnnotation(holderObject.getClass(), StellarConfigurationList.class); + + for (Field f : holderList) { + Object potentialChild = FieldUtils.readField(f, holderObject,true); + if (potentialChild == null) { + break; + } + ExpressionConfigurationHolder child = new ExpressionConfigurationHolder(getFullName(), f.getName(), + potentialChild); + + child.discover(); + children.add(child); + } + for (Field f : holderListList) { + String thisFullName = String + .format("%s/%s", getFullName(), f.getAnnotation(StellarConfigurationList.class).name()); + Object potentialChild = FieldUtils.readField(f, holderObject,true); + if (potentialChild == null) { + break; + } + if (!Iterable.class.isAssignableFrom(potentialChild.getClass())) { + break; + } + Iterable it = (Iterable) FieldUtils.readField(f, holderObject, true); + int index = 0; + for (Object thisHolderObject : it) { + if (thisHolderObject == null) { + break; + } + ExpressionConfigurationHolder child = new ExpressionConfigurationHolder(thisFullName, String.valueOf(index), + thisHolderObject); + index++; + child.discover(); + children.add(child); + } + } + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarConfiguredStatementReporter.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java similarity index 60% rename from metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarConfiguredStatementReporter.java rename to metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java index aafd544d2f..997a70b35e 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarConfiguredStatementReporter.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java @@ -16,33 +16,31 @@ * limitations under the License. */ -package org.apache.metron.stellar.common; +package org.apache.metron.stellar.common.utils.validation; import java.util.List; import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementVisitor.ErrorConsumer; import org.atteo.classindex.IndexSubclasses; /** - * StellarConfiguredStatementProviders are used provide stellar statements - * and the context around those statements to the caller + * StellarConfigurationProvider are used provide stellar statements + * and the context around those statements to the caller. */ @IndexSubclasses -public interface StellarConfiguredStatementReporter { +public interface StellarConfigurationProvider { /** - * The Name of this reporter + * The Name of this reporter. * @return String */ String getName(); - public interface StatementReportVisitor{ - void visit(List contextNames, String statement); - } - - public interface ConfigReportErrorConsumer { - void consume(List contextNames, Exception e); - } - - void vist(CuratorFramework client, StatementReportVisitor visitor, ConfigReportErrorConsumer errorConsumer) throws Exception; + /** + * Returns a list of all known ExpressionConfigurationHolders. + * @param client The Zookeeper client + * @return List of ExpressionConfigurationHolder + */ + List provideConfigurations(CuratorFramework client, ErrorConsumer errorConsumer); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementVisitor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementVisitor.java new file mode 100644 index 0000000000..99f20a70d3 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementVisitor.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +/** + * StellarConfiguredStatementProviders are used provide stellar statements + * and the context around those statements to the caller + */ +public interface StellarConfiguredStatementVisitor { + + public interface StatementVisitor { + void visit(String path, String statement); + } + + public interface ErrorConsumer { + void consume(String path, Exception e); + } + + void visit(StatementVisitor visitor, ErrorConsumer errorConsumer) throws Exception; + +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java index 96343ab713..b8c864380b 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java @@ -23,19 +23,20 @@ import static org.apache.metron.stellar.common.shell.StellarShell.ERROR_PROMPT; import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; import java.util.Set; import org.apache.commons.lang.NullArgumentException; import org.apache.curator.framework.CuratorFramework; -import org.apache.metron.stellar.common.StellarConfiguredStatementReporter; import org.apache.metron.stellar.common.StellarProcessor; import org.atteo.classindex.ClassIndex; -public class StellarZookeeperBasedValidator implements StellarValidator{ +public class StellarZookeeperBasedValidator implements StellarValidator { private CuratorFramework client; public StellarZookeeperBasedValidator(CuratorFramework client) throws NullArgumentException { - if(client == null) { + if (client == null) { throw new NullArgumentException("client"); } this.client = client; @@ -44,61 +45,71 @@ public StellarZookeeperBasedValidator(CuratorFramework client) throws NullArgume @Override public void validate(LineWriter writer) { - // discover all the StellarConfiguredStatementReporters - Set reporterSet = new HashSet<>(); + // discover all the StellarConfigurationProvider + Set providerSet = new HashSet<>(); - for (Class c : ClassIndex.getSubclasses(StellarConfiguredStatementReporter.class, + for (Class c : ClassIndex.getSubclasses(StellarConfigurationProvider.class, Thread.currentThread().getContextClassLoader())) { - boolean isAssignable = StellarConfiguredStatementReporter.class.isAssignableFrom(c); + boolean isAssignable = StellarConfigurationProvider.class.isAssignableFrom(c); if (isAssignable) { try { - StellarConfiguredStatementReporter reporter = StellarConfiguredStatementReporter.class + StellarConfigurationProvider reporter = StellarConfigurationProvider.class .cast(c.getConstructor().newInstance()); - reporterSet.add(reporter); + providerSet.add(reporter); } catch (Exception e) { - writer.write(ERROR_PROMPT + " Reporter: " + c.getCanonicalName() + " not valid, skipping"); + writer + .write(ERROR_PROMPT + " Provider: " + c.getCanonicalName() + " not valid, skipping"); } } } - writer.write(String.format("Discovered %d reporters", reporterSet.size())); + writer.write(String.format("Discovered %d providers", providerSet.size())); writer.write("Visiting all configurations. ThreatTriage rules are checked when loading the " - + "configuration, thus an invalid ThreatTriage rule will fail the entire Enrichement Configuration."); - if (reporterSet.size() > 0) { - reporterSet.forEach((r) -> writer.write(r.getName() + " ")); + + "configuration, thus an invalid ThreatTriage rule will fail the entire Enrichment Configuration."); + if (providerSet.size() > 0) { + providerSet.forEach((r) -> writer.write(r.getName() + " ")); writer.write(""); } else { return; } - reporterSet.forEach((r) -> { - writer.write("Visiting " + r.getName()); + providerSet.forEach((r) -> { + writer.write("Requesting configurations from " + r.getName()); try { - r.vist(client, ((contextNames, statement) -> { - // the names taken together are the identifier for this - // statement - String name = String.join("->", contextNames); + List holders = r + .provideConfigurations(client, (pathName, exception) -> { + writer.write(ERROR_PROMPT + String + .format("Configuration %s is not valid, please review", pathName)); + }); - writer.write("=================================================="); - writer.write("validating " + name); + writer.write(String.format("%s provided %d configurations", r.getName(), holders.size())); + holders.forEach((h) -> { try { - if (StellarProcessor.compile(statement) == null) { - writer.write( - ERROR_PROMPT + String.format("Statement: %s is not valid, please review", name)); - writer.write(ERROR_PROMPT + String.format("Statement: %s ", statement)); - } - } catch (RuntimeException e) { - writer.write(ERROR_PROMPT + "Error Visiting " + name); - writer.write(e.getMessage()); - writer.write("--"); - writer.write(ERROR_PROMPT + ": " + statement); + h.discover(); + h.visit((path, statement) -> { + writer.write("=================================================="); + writer.write(String.format("validating %s", path)); + try { + if (StellarProcessor.compile(statement) == null) { + writer.write(ERROR_PROMPT + String + .format("Statement: %s is not valid, please review", path)); + writer.write(ERROR_PROMPT + String.format("Statement: %s ", statement)); + } + } catch (RuntimeException e) { + writer.write(ERROR_PROMPT + "Error Visiting " + path); + writer.write(e.getMessage()); + writer.write("--"); + writer.write(ERROR_PROMPT + ": " + statement); + } + writer.write("=================================================="); + }, (path, error) -> { + writer.write(ERROR_PROMPT + String + .format("Configuration %s is not valid, please review", path)); + }); + } catch (Exception e) { + writer.write(ERROR_PROMPT + " " + e.getMessage()); } - writer.write("=================================================="); - }), (contextNames, exception) -> { - String name = String.join("->", contextNames); - writer.write( - ERROR_PROMPT + String.format("Configuration %s is not valid, please review", name)); }); } catch (Exception e) { writer.write(ERROR_PROMPT + "Error Visiting " + r.getName()); diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java new file mode 100644 index 0000000000..277fe02fc7 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface StellarConfiguration { +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java new file mode 100644 index 0000000000..94b401855c --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface StellarConfigurationList { + String name(); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java new file mode 100644 index 0000000000..ff9ebe500a --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface StellarExpressionField { + String name(); +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java new file mode 100644 index 0000000000..841d2db3be --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface StellarExpressionList { + String name(); + +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java new file mode 100644 index 0000000000..d853704cb1 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface StellarExpressionMap { + + /** + * The Name to give to the map + * + * @return String of the name + */ + String name() default "default"; + + /** + * A map may be a StellarExpressionMap based on the type + * of another field, this is that field's name + * + * @return Field Name or empty String + */ + String qualify_with_field() default ""; + + /** + * A map may be a StellarExpressionMap based on the type + * of another field, this is that type + * + * @return Class + */ + Class qualify_with_field_type() default Error.class; + + + /** + * Some maps, may 'hold' the real map, + * These are the key to get that map. + * There are multiple in case there are multiple nested level + * This does not support multiple nested 'peer' maps + * + * @return String key + */ + String[] inner_map_keys() default {}; +} \ No newline at end of file From 70de632ee583b45c028b23d8305d76e4b6bc70c5 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Fri, 1 Dec 2017 11:59:39 -0500 Subject: [PATCH 06/14] fix imports --- .../field/transformation/StellarTransformation.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java index b9c97f5329..bbdacec1ad 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java @@ -18,14 +18,16 @@ package org.apache.metron.common.field.transformation; -import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; -import org.apache.metron.stellar.dsl.VariableResolver; -import org.apache.metron.stellar.common.StellarProcessor; - -import java.util.*; public class StellarTransformation implements FieldTransformation { @Override From 8726a15a3db35bf24408e723ec069a391df16820 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Fri, 1 Dec 2017 14:53:52 -0500 Subject: [PATCH 07/14] small refactor and javadoc work --- ...Reporter.java => ConfigurationProvider.java} | 16 +++------------- ...Test.java => ConfigurationProviderTest.java} | 14 +++++++------- .../ExpressionConfigurationHolder.java | 17 ++++++++++++++++- .../StellarConfigurationProvider.java | 2 +- .../annotations/StellarConfiguration.java | 11 +++++++++++ .../annotations/StellarConfigurationList.java | 8 ++++++++ .../annotations/StellarExpressionField.java | 11 +++++++++++ .../annotations/StellarExpressionList.java | 9 +++++++++ .../annotations/StellarExpressionMap.java | 12 ++++++++++++ 9 files changed, 78 insertions(+), 22 deletions(-) rename metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/{StellarStatementReporter.java => ConfigurationProvider.java} (88%) rename metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/{StellarStatementReporterTest.java => ConfigurationProviderTest.java} (94%) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java similarity index 88% rename from metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java rename to metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java index 4491234229..2726f5cc91 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/StellarStatementReporter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java @@ -39,12 +39,12 @@ import org.apache.zookeeper.KeeperException.NoNodeException; /** - * StellarStatementReporter is used to report all of the configured / deployed Stellar statements in + * {@code ConfigurationProvider} is used to report all of the configured / deployed Stellar statements in * the system. */ -public class StellarStatementReporter implements StellarConfigurationProvider { +public class ConfigurationProvider implements StellarConfigurationProvider { - public StellarStatementReporter() { + public ConfigurationProvider() { } @Override @@ -117,16 +117,6 @@ private void visitEnrichmentConfigs(CuratorFramework client, String.format("%s/%s", getName(), ENRICHMENT.toString()), child, sensorEnrichmentConfig); holders.add(holder); - /* - EnrichmentConfig enrichmentConfig; - enrichmentConfig = sensorEnrichmentConfig.getEnrichment(); - visitEnrichmentConfig(child, Type.ENRICHMENT, enrichmentConfig, visitor, errorConsumer); - enrichmentConfig = sensorEnrichmentConfig.getThreatIntel(); - visitEnrichmentConfig(child, Type.THREAT_INTEL, enrichmentConfig, visitor, errorConsumer); - final ThreatTriageConfig threatTriageConfig = sensorEnrichmentConfig.getThreatIntel() - .getTriageConfig(); - visitEnrichmentThreatTriageConfigs(child, threatTriageConfig, visitor, errorConsumer); - */ } catch (Exception e) { errorConsumer .consume(String.format("%s/%s/%s", getName(), ENRICHMENT.toString(), child), e); diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java similarity index 94% rename from metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java rename to metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java index 84f72058c5..2c554ebc3e 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarStatementReporterTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java @@ -29,7 +29,7 @@ import org.junit.Before; import org.junit.Test; -public class StellarStatementReporterTest { +public class ConfigurationProviderTest { private TestingServer testZkServer; private String zookeeperUrl; @@ -259,8 +259,8 @@ public void testProvideAllConfiguredNodes() throws Exception { zookeeperUrl); ConfigurationsUtils.writeProfilerConfigToZookeeper(profilerConfig.getBytes(), client); - StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); - List holders = stellarStatementReporter + ConfigurationProvider configurationProvider = new ConfigurationProvider(); + List holders = configurationProvider .provideConfigurations(client, (names, error) -> Assert.assertTrue("Should not get errors", false)); holders.forEach((h) -> { @@ -289,8 +289,8 @@ public void testErrorCalledWithBadParserConfig() throws Exception { ConfigurationsUtils .writeToZookeeper(ConfigurationType.PARSER.getZookeeperRoot() + "/" + "squid", badParserConfig.getBytes(), client); - StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); - stellarStatementReporter.provideConfigurations(client, (names, error) -> { + ConfigurationProvider configurationProvider = new ConfigurationProvider(); + configurationProvider.provideConfigurations(client, (names, error) -> { Assert.assertTrue("Should get errors", true); }); } @@ -300,8 +300,8 @@ public void testExceptionThrownGetsBadThreatConfig() throws Exception { ConfigurationsUtils .writeSensorEnrichmentConfigToZookeeper("snort", badEnrichmentConfig.getBytes(), zookeeperUrl); - StellarStatementReporter stellarStatementReporter = new StellarStatementReporter(); - stellarStatementReporter.provideConfigurations(client, + ConfigurationProvider configurationProvider = new ConfigurationProvider(); + configurationProvider.provideConfigurations(client, (names, error) -> Assert.assertTrue("Should not get errors", false)); } } \ No newline at end of file diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java index 70ee72b2c3..13d808e4d2 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java @@ -18,7 +18,6 @@ package org.apache.metron.stellar.common.utils.validation; -import java.lang.annotation.AnnotationTypeMismatchException; import java.lang.annotation.IncompleteAnnotationException; import java.lang.reflect.Field; import java.util.LinkedList; @@ -33,6 +32,22 @@ import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionList; import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; +/** + * {@code ExpressionConfigurationHolder} is a wrapper class for Objects that represent configurations + * that have been annotated with the annotations from the validation.annotations package. + * + * Holders understand how to discover the stellar expressions contained in them by understanding how + * to evaluate thier own annotated fields. + * + * No knowledge of the implementation of the configuration + * Object itself is required by this class. Instead it evaluates it's own member fields for + * {@code StellarExpressionField}, {@code StellarExpressionList}, {@code StellarExpressionMap} instances. + * + * Complex types annotated by {@code StellarConfiguration} or {@code StellarConfigurationList} are + * treated as children of this class. + * + * + */ public class ExpressionConfigurationHolder implements StellarConfiguredStatementVisitor { private Object holderObject; diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java index 997a70b35e..7aee59f5e9 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java @@ -24,7 +24,7 @@ import org.atteo.classindex.IndexSubclasses; /** - * StellarConfigurationProvider are used provide stellar statements + * {@code StellarConfigurationProvider} are used provide Stellar statements * and the context around those statements to the caller. */ @IndexSubclasses diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java index 277fe02fc7..bb3c5299b7 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java @@ -22,6 +22,17 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +/** + * {@code StellarConfiguration} marks a field or class has a container + * for some stellar configuration or expression. + * + * That expression may be in a field such as a String, or a Map, or list or other complex type. + * Each of those types in turn should be annotated with the appropriate annotation. + * + * Other complex types that are fields of a {@code StellarConfiguration} , should likewise be annotated as + * {@code StellarConfiguration}. + * + */ @Documented @Retention(RetentionPolicy.RUNTIME) public @interface StellarConfiguration { diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java index 94b401855c..99fc9275d7 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java @@ -22,8 +22,16 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +/** + * {@code StellarConfigurationList} marks a field as being a list + * of objects that represent {@code StellarConfiguration} annotated objects. + */ @Documented @Retention(RetentionPolicy.RUNTIME) public @interface StellarConfigurationList { + /** + * The Name to be applied to this field. + * @return the Name + */ String name(); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java index ff9ebe500a..3f1d881d71 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java @@ -22,8 +22,19 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +/** + * {@code StellarExpressionField} marks a field + * as being a Stellar expression. + * + * It's toString() method will be called to provide this String. + */ @Documented @Retention(RetentionPolicy.RUNTIME) public @interface StellarExpressionField { + + /** + * The Name to be applied to this field. + * @return the Name + */ String name(); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java index 841d2db3be..62c5611794 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java @@ -22,9 +22,18 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +/** + * {@code StellarExpressionList} is applied to a field + * that is an {@code Iterable} of objects that are Stellar expressions, + * Such that calling toString() on that object yields a string of a Stellar expression + */ @Documented @Retention(RetentionPolicy.RUNTIME) public @interface StellarExpressionList { + /** + * The Name to be applied to this field. + * @return the Name + */ String name(); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java index d853704cb1..bd6c043205 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java @@ -22,6 +22,18 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +/** + * {@code StellarExpressionMap} is applied to + * a {@code Map} which contains Stellar expressions as the values, such + * that calling .toString() on a value of this map yields a Stellar expression. + * + * The key is used as the of the expression. + * + * It is possible for a {@code Map} to contain other maps or complex objects, + * thus this annotation contains properties that give information on evaluation of the {@code Map} + * should it be complex and contain nested maps. + * + */ @Documented @Retention(RetentionPolicy.RUNTIME) public @interface StellarExpressionMap { From a6a9a4e5d558209175a8e5d2fa532c845efa830d Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Sun, 3 Dec 2017 08:22:50 -0500 Subject: [PATCH 08/14] format and javadoc --- .../configuration/ConfigurationProvider.java | 5 +- .../ExpressionConfigurationHolder.java | 124 +++++++++++++----- .../annotations/StellarConfiguration.java | 3 + .../annotations/StellarConfigurationList.java | 3 + .../annotations/StellarExpressionField.java | 3 + .../annotations/StellarExpressionList.java | 3 + .../annotations/StellarExpressionMap.java | 17 ++- 7 files changed, 117 insertions(+), 41 deletions(-) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java index 2726f5cc91..0aa22018e3 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java @@ -44,6 +44,9 @@ */ public class ConfigurationProvider implements StellarConfigurationProvider { + /** + * Default constructor. + */ public ConfigurationProvider() { } @@ -102,7 +105,7 @@ private void visitEnrichmentConfigs(CuratorFramework client, byte[] data = client.getData().forPath(ENRICHMENT.getZookeeperRoot() + "/" + child); // Certain parts of the SensorEnrichmentConfig do Stellar Verification on their // own as part of deserialization, where the bean spec will call the setter, which has - // been wired with stellar verification calls. + // been wired with stellar verification calls. There is no avoiding this. // // In cases where those parts of the config are in fact the parts that have invalid // Stellar statements, we will fail during the JSON load before we get to ANY config diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java index 13d808e4d2..3ee9324a42 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java @@ -33,20 +33,24 @@ import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; /** - * {@code ExpressionConfigurationHolder} is a wrapper class for Objects that represent configurations - * that have been annotated with the annotations from the validation.annotations package. - * + *

+ * {@code ExpressionConfigurationHolder} is a wrapper class for Objects that represent + * configurations that have been annotated with the annotations from the validation.annotations + * package. + *

+ *

* Holders understand how to discover the stellar expressions contained in them by understanding how * to evaluate thier own annotated fields. - * + *

+ *

* No knowledge of the implementation of the configuration * Object itself is required by this class. Instead it evaluates it's own member fields for * {@code StellarExpressionField}, {@code StellarExpressionList}, {@code StellarExpressionMap} instances. - * + *

+ *

* Complex types annotated by {@code StellarConfiguration} or {@code StellarConfigurationList} are * treated as children of this class. - * - * + *

*/ public class ExpressionConfigurationHolder implements StellarConfiguredStatementVisitor { @@ -57,8 +61,15 @@ public class ExpressionConfigurationHolder implements StellarConfiguredStatement private List children = new LinkedList<>(); private List expressionList; private List expressionListList; - private List expresionMapList; + private List expressionMapList; + /** + * Constructs a new {@code ExpressionConfigurationHolder}. + * + * @param parentName the name of the parent object of this holder + * @param name the name of to be used for this holder + * @param holderObject the object being held + */ public ExpressionConfigurationHolder(String parentName, String name, Object holderObject) { if (holderObject == null) { throw new NullArgumentException("holderObject"); @@ -75,18 +86,39 @@ public ExpressionConfigurationHolder(String parentName, String name, Object hold this.holderObject = holderObject; } + /** + * Returns the {@code Object} being held. + * + * @return {@code Object} + */ public Object getHolderObject() { return holderObject; } + /** + * Returns the name for this {@code ExpressionConfigurationHolder}. + * + * @return String value of the name + */ public String getName() { return name; } + /** + * Returns the name of the parent of this {@code ExpressionConfigurationHolder}. + * + * @return String value of the name + */ public String getParentName() { return parentName; } + /** + * Returns the full name of this {@code ExpressionConfigurationHolder}. + * This represents the name taken together with the parentName, path delimeted by a "/" + * + * @return String value of the name + */ public String getFullName() { return fullName; } @@ -99,26 +131,24 @@ public void visit(StatementVisitor visitor, ErrorConsumer errorConsumer) { visitChilden(visitor, errorConsumer); } - private void visitExpressions(StatementVisitor visitor, - ErrorConsumer errorConsumer) { + private void visitExpressions(StatementVisitor visitor, ErrorConsumer errorConsumer) { expressionList.forEach((f) -> { String thisFullName = String .format("%s/%s", getFullName(), f.getAnnotation(StellarExpressionField.class).name()); try { - Object thisExpressionObject = FieldUtils.readField(f, holderObject, true); + Object thisExpressionObject = FieldUtils.readField(f, holderObject, true); if (thisExpressionObject == null || StringUtils.isEmpty(thisExpressionObject.toString())) { return; } - visitExpression(thisFullName,thisExpressionObject.toString(), visitor, - errorConsumer); + visitExpression(thisFullName, thisExpressionObject.toString(), visitor, errorConsumer); } catch (IllegalAccessException e) { errorConsumer.consume(thisFullName, e); } }); } - private void visitExpression(String expressionName, String expression, - StatementVisitor visitor, ErrorConsumer errorConsumer) { + private void visitExpression(String expressionName, String expression, StatementVisitor visitor, + ErrorConsumer errorConsumer) { if (StringUtils.isEmpty(expression)) { return; } @@ -138,7 +168,7 @@ private void visitLists(StatementVisitor visitor, ErrorConsumer errorConsumer) { if (possibleIterable == null) { return; } - Iterable it = (Iterable)possibleIterable; + Iterable it = (Iterable) possibleIterable; int index = 0; for (Object expressionObject : it) { String expressionFullName = String.format("%s/%s", thisFullName, index); @@ -152,19 +182,21 @@ private void visitLists(StatementVisitor visitor, ErrorConsumer errorConsumer) { @SuppressWarnings("unchecked") private void visitMaps(StatementVisitor visitor, ErrorConsumer errorConsumer) { - expresionMapList.forEach((f) -> { + expressionMapList.forEach((f) -> { String thisFullName = String .format("%s/%s", getFullName(), f.getAnnotation(StellarExpressionMap.class).name()); try { // if we have configured a qualifying field as a type of flag we need to check it // for example, if StellarFoo.class.isAssignableFrom(this.foo.getClass()) then // bar is a StellarExpressionMap - if (!StringUtils.isEmpty(f.getAnnotation(StellarExpressionMap.class).qualify_with_field())) { + if (!StringUtils + .isEmpty(f.getAnnotation(StellarExpressionMap.class).qualify_with_field())) { String fieldName = f.getAnnotation(StellarExpressionMap.class).qualify_with_field(); Class type = f.getAnnotation(StellarExpressionMap.class).qualify_with_field_type(); Object theObject = FieldUtils.readField(holderObject, fieldName, true); if (theObject == null) { - errorConsumer.consume(thisFullName, new IncompleteAnnotationException(StellarExpressionMap.class,"fieldName")); + errorConsumer.consume(thisFullName, + new IncompleteAnnotationException(StellarExpressionMap.class, "fieldName")); return; } if (!type.isAssignableFrom(theObject.getClass())) { @@ -185,23 +217,25 @@ private void visitMaps(StatementVisitor visitor, ErrorConsumer errorConsumer) { if (innerObject == null) { return; } - fullName = String.format("%s/%s",fullName,key); - if(!Map.class.isAssignableFrom(innerObject.getClass())) { - errorConsumer.consume(fullName, new Exception("The annotation specified an inner map that was not a map")); + fullName = String.format("%s/%s", fullName, key); + if (!Map.class.isAssignableFrom(innerObject.getClass())) { + errorConsumer.consume(fullName, + new Exception("The annotation specified an inner map that was not a map")); } - map = (Map)innerObject; + map = (Map) innerObject; } } map.forEach((k, v) -> { String mapKeyFullName = String.format("%s/%s", thisFullName, k.toString()); if (Map.class.isAssignableFrom(v.getClass())) { - Map innerMap = (Map)v; - innerMap.forEach((ik,iv) -> { + Map innerMap = (Map) v; + innerMap.forEach((ik, iv) -> { if (iv == null) { return; } - visitExpression(String.format("%s/%s",mapKeyFullName,ik.toString()),iv.toString(), visitor, errorConsumer); + visitExpression(String.format("%s/%s", mapKeyFullName, ik.toString()), iv.toString(), + visitor, errorConsumer); }); return; } @@ -213,17 +247,33 @@ private void visitMaps(StatementVisitor visitor, ErrorConsumer errorConsumer) { }); } - private void visitChilden(StatementVisitor visitor, - ErrorConsumer errorConsumer) { - children.forEach((c) -> c.visit(visitor,errorConsumer)); + private void visitChilden(StatementVisitor visitor, ErrorConsumer errorConsumer) { + children.forEach((c) -> c.visit(visitor, errorConsumer)); } + /** + * When {@code discover} is called, the {@code ExpressionConfigurationHolder} evaluates it's + * members such that it discovers the fields annotated with any of the validation.annotations. + *

+ * The annotated objects found are grouped by annotation type. Any objects annotated as + * {@code StellarConfiguration} will have new {@code ExpressionConfigurationHolder} objects + * created for them, and these objects will be added as children of this + * {@code ExpressionConfigurationHolder}. + *

+ *

+ * Discovery uses the {@code org.apache.commons.lang3.reflect.FieldUtils} classes for finding + * and reading the values of annotated fields. + * This class uses fields as opposed to methods to avoid an possible mutation or side effects + * from the discovery operation. + *

+ * The same for the contents of an object annotated as a {@code StellarConfigurationList}. + */ public void discover() throws IllegalAccessException { expressionList = FieldUtils .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionField.class); expressionListList = FieldUtils .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionList.class); - expresionMapList = FieldUtils + expressionMapList = FieldUtils .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionMap.class); List holderList = FieldUtils .getFieldsListWithAnnotation(holderObject.getClass(), StellarConfiguration.class); @@ -231,20 +281,22 @@ public void discover() throws IllegalAccessException { .getFieldsListWithAnnotation(holderObject.getClass(), StellarConfigurationList.class); for (Field f : holderList) { - Object potentialChild = FieldUtils.readField(f, holderObject,true); + Object potentialChild = FieldUtils.readField(f, holderObject, true); if (potentialChild == null) { break; } - ExpressionConfigurationHolder child = new ExpressionConfigurationHolder(getFullName(), f.getName(), - potentialChild); + + ExpressionConfigurationHolder child = new ExpressionConfigurationHolder(getFullName(), + f.getName(), potentialChild); child.discover(); children.add(child); } + for (Field f : holderListList) { String thisFullName = String .format("%s/%s", getFullName(), f.getAnnotation(StellarConfigurationList.class).name()); - Object potentialChild = FieldUtils.readField(f, holderObject,true); + Object potentialChild = FieldUtils.readField(f, holderObject, true); if (potentialChild == null) { break; } @@ -257,8 +309,8 @@ public void discover() throws IllegalAccessException { if (thisHolderObject == null) { break; } - ExpressionConfigurationHolder child = new ExpressionConfigurationHolder(thisFullName, String.valueOf(index), - thisHolderObject); + ExpressionConfigurationHolder child = new ExpressionConfigurationHolder(thisFullName, + String.valueOf(index), thisHolderObject); index++; child.discover(); children.add(child); diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java index bb3c5299b7..3a861dd682 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfiguration.java @@ -19,8 +19,10 @@ package org.apache.metron.stellar.common.utils.validation.annotations; import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; /** * {@code StellarConfiguration} marks a field or class has a container @@ -35,5 +37,6 @@ */ @Documented @Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) public @interface StellarConfiguration { } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java index 99fc9275d7..067695ef14 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarConfigurationList.java @@ -19,8 +19,10 @@ package org.apache.metron.stellar.common.utils.validation.annotations; import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; /** * {@code StellarConfigurationList} marks a field as being a list @@ -28,6 +30,7 @@ */ @Documented @Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) public @interface StellarConfigurationList { /** * The Name to be applied to this field. diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java index 3f1d881d71..c656787ab2 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionField.java @@ -19,8 +19,10 @@ package org.apache.metron.stellar.common.utils.validation.annotations; import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; /** * {@code StellarExpressionField} marks a field @@ -30,6 +32,7 @@ */ @Documented @Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) public @interface StellarExpressionField { /** diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java index 62c5611794..b19292dfd5 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionList.java @@ -19,8 +19,10 @@ package org.apache.metron.stellar.common.utils.validation.annotations; import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; /** * {@code StellarExpressionList} is applied to a field @@ -29,6 +31,7 @@ */ @Documented @Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) public @interface StellarExpressionList { /** * The Name to be applied to this field. diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java index bd6c043205..d914b76e14 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java @@ -19,15 +19,17 @@ package org.apache.metron.stellar.common.utils.validation.annotations; import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; /** * {@code StellarExpressionMap} is applied to * a {@code Map} which contains Stellar expressions as the values, such * that calling .toString() on a value of this map yields a Stellar expression. * - * The key is used as the of the expression. + * The key is used as the name the expression. * * It is possible for a {@code Map} to contain other maps or complex objects, * thus this annotation contains properties that give information on evaluation of the {@code Map} @@ -36,6 +38,7 @@ */ @Documented @Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD,ElementType.TYPE}) public @interface StellarExpressionMap { /** @@ -47,7 +50,9 @@ /** * A map may be a StellarExpressionMap based on the type - * of another field, this is that field's name + * of another field, this is that field's name. + * + * {@code} qualify_with_field_type} is the type of that field * * @return Field Name or empty String */ @@ -55,7 +60,9 @@ /** * A map may be a StellarExpressionMap based on the type - * of another field, this is that type + * of another field, this is that type of that field. + * + * {@code qualify_with_field} is the name of the field. * * @return Class */ @@ -63,7 +70,8 @@ /** - * Some maps, may 'hold' the real map, + * Some maps, may 'hold' the real map as a value. In that case the outer map should still be + * annotated, and this property used to define the 'key route' needed to obtain the map. * These are the key to get that map. * There are multiple in case there are multiple nested level * This does not support multiple nested 'peer' maps @@ -71,4 +79,5 @@ * @return String key */ String[] inner_map_keys() default {}; + } \ No newline at end of file From 3f12c2dace1157ffd4e870df39864b61b71c1270 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Sun, 3 Dec 2017 15:31:30 -0500 Subject: [PATCH 09/14] refactor name and tests --- .../configuration/ConfigurationProvider.java | 9 +- .../ExpressionConfigurationHolder.java | 13 +-- .../StellarConfigurationProvider.java | 2 +- ... StellarConfiguredStatementContainer.java} | 3 +- .../ExpressionConfigurationHolderTest.java | 68 +++++++++++++++ .../validation/TestChildConfigObject.java | 32 +++++++ .../utils/validation/TestConfigObject.java | 84 +++++++++++++++++++ 7 files changed, 196 insertions(+), 15 deletions(-) rename metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/{StellarConfiguredStatementVisitor.java => StellarConfiguredStatementContainer.java} (93%) create mode 100644 metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolderTest.java create mode 100644 metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestChildConfigObject.java create mode 100644 metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java index 0aa22018e3..a4d8528197 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java @@ -19,23 +19,16 @@ import static org.apache.metron.common.configuration.ConfigurationType.PARSER; import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import org.apache.curator.framework.CuratorFramework; -import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; -import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; import org.apache.metron.common.configuration.profiler.ProfileConfig; -import org.apache.metron.common.configuration.profiler.ProfileResult; -import org.apache.metron.common.configuration.profiler.ProfileResultExpressions; -import org.apache.metron.common.configuration.profiler.ProfileTriageExpressions; import org.apache.metron.common.configuration.profiler.ProfilerConfig; -import org.apache.metron.common.field.transformation.StellarTransformation; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.stellar.common.utils.validation.ExpressionConfigurationHolder; import org.apache.metron.stellar.common.utils.validation.StellarConfigurationProvider; -import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementVisitor.ErrorConsumer; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer; import org.apache.zookeeper.KeeperException.NoNodeException; /** diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java index 3ee9324a42..c7f7200178 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java @@ -52,7 +52,7 @@ * treated as children of this class. *

*/ -public class ExpressionConfigurationHolder implements StellarConfiguredStatementVisitor { +public class ExpressionConfigurationHolder implements StellarConfiguredStatementContainer { private Object holderObject; private String name; @@ -173,6 +173,7 @@ private void visitLists(StatementVisitor visitor, ErrorConsumer errorConsumer) { for (Object expressionObject : it) { String expressionFullName = String.format("%s/%s", thisFullName, index); visitExpression(expressionFullName, expressionObject.toString(), visitor, errorConsumer); + index++; } } catch (IllegalAccessException e) { errorConsumer.consume(thisFullName, e); @@ -217,17 +218,18 @@ private void visitMaps(StatementVisitor visitor, ErrorConsumer errorConsumer) { if (innerObject == null) { return; } - fullName = String.format("%s/%s", fullName, key); + thisFullName = String.format("%s/%s", thisFullName, key); if (!Map.class.isAssignableFrom(innerObject.getClass())) { - errorConsumer.consume(fullName, + errorConsumer.consume(thisFullName, new Exception("The annotation specified an inner map that was not a map")); } map = (Map) innerObject; } } + final String finalName = thisFullName; map.forEach((k, v) -> { - String mapKeyFullName = String.format("%s/%s", thisFullName, k.toString()); + String mapKeyFullName = String.format("%s/%s", finalName, k.toString()); if (Map.class.isAssignableFrom(v.getClass())) { Map innerMap = (Map) v; innerMap.forEach((ik, iv) -> { @@ -268,7 +270,8 @@ private void visitChilden(StatementVisitor visitor, ErrorConsumer errorConsumer) *

* The same for the contents of an object annotated as a {@code StellarConfigurationList}. */ - public void discover() throws IllegalAccessException { + @Override + public void discover() throws Exception { expressionList = FieldUtils .getFieldsListWithAnnotation(holderObject.getClass(), StellarExpressionField.class); expressionListList = FieldUtils diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java index 7aee59f5e9..645d7e43bf 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java @@ -20,7 +20,7 @@ import java.util.List; import org.apache.curator.framework.CuratorFramework; -import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementVisitor.ErrorConsumer; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer; import org.atteo.classindex.IndexSubclasses; /** diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementVisitor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java similarity index 93% rename from metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementVisitor.java rename to metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java index 99f20a70d3..2e5bd4ed40 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementVisitor.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java @@ -22,7 +22,7 @@ * StellarConfiguredStatementProviders are used provide stellar statements * and the context around those statements to the caller */ -public interface StellarConfiguredStatementVisitor { +public interface StellarConfiguredStatementContainer { public interface StatementVisitor { void visit(String path, String statement); @@ -33,5 +33,6 @@ public interface ErrorConsumer { } void visit(StatementVisitor visitor, ErrorConsumer errorConsumer) throws Exception; + void discover() throws Exception; } diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolderTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolderTest.java new file mode 100644 index 0000000000..dc6e62a303 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolderTest.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.util.LinkedList; +import java.util.List; +import org.junit.Assert; +import org.junit.Test; + +public class ExpressionConfigurationHolderTest { + + @Test + @SuppressWarnings("unchecked") + public void visitAndDiscover() throws Exception { + List discoveredPaths = new LinkedList<>(); + List expectedPaths = new LinkedList(){{ + add("unit tests/test config/the field"); + add("unit tests/test config/the list of stellar strings/0"); + add("unit tests/test config/the list of stellar strings/1"); + add("unit tests/test config/plain old map/stellar_map_item"); + add("unit tests/test config/map two deep/level1/level2/nested stellar key"); + add("unit tests/test config/map depending on field type/stellar_key"); + add("unit tests/test config/child/child field"); + add("unit tests/test config/list of holders/0/child field"); + add("unit tests/test config/list of holders/1/child field"); + }}; + TestConfigObject testSubject = new TestConfigObject(); + ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder("unit tests", + "test config", testSubject); + + holder.discover(); + holder.visit(((path, statement) -> { + discoveredPaths.add(path); + }),((path, e) -> { + Assert.assertTrue("Should not get errors", false); + })); + + Assert.assertFalse(discoveredPaths.contains("unit tests/test config/map depending on field type but wrong/stellar_key")); + + Assert.assertEquals(discoveredPaths.size(),expectedPaths.size()); + expectedPaths.forEach((p) -> { + if(!discoveredPaths.contains(p)) { + Assert.assertTrue(String.format("discovered missing %s", p), false); + } + }); + + discoveredPaths.removeAll(expectedPaths); + + Assert.assertTrue(discoveredPaths.isEmpty()); + } + +} \ No newline at end of file diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestChildConfigObject.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestChildConfigObject.java new file mode 100644 index 0000000000..9318bea949 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestChildConfigObject.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; + +@StellarConfiguration +public class TestChildConfigObject { + @StellarExpressionField(name = "child field") + private String field; + + public TestChildConfigObject(Number number){ + field = String.format("TO_STRING(%d)",number.intValue()); + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java new file mode 100644 index 0000000000..877aa62036 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionList; +import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap; + +@StellarConfiguration +@SuppressWarnings("unchecked") +public class TestConfigObject { + + // a field + @StellarExpressionField(name = "the field") + private String stellar = "TO_UPPER('stellarexpressionfield')"; + + // a list of stellar + @StellarExpressionList(name = "the list of stellar strings") + private List stellarList = new LinkedList() {{ + add("TO_UPPER('list stellar item one')"); + add("TO_UPPER('list stellar item two')"); + }}; + + + // a map + @StellarExpressionMap(name = "plain old map") + private Map plainMap = new HashMap() {{ + put("stellar_map_item", "TO_UPPER('SMI')"); + }}; + + + // a map with the 'real' map burried 2 layers in + @StellarExpressionMap(name = "map two deep", inner_map_keys = {"level1", "level2"}) + private Map mapTwoDeep = new HashMap(){{ + put("level1",new HashMap(){{ + put("level2",new HashMap(){{ + put("nested stellar key", "TO_UPPER('dig down deep')"); + }}); + }}); + }}; + + @StellarExpressionMap(name = "map depending on field type", qualify_with_field = "qualifier_field", qualify_with_field_type = java.lang.Integer.class) + private Map dependsMap = new HashMap(){{ + put("stellar_key", "TO_UPPER('it was int')"); + }}; + + @StellarExpressionMap(name = "map depending on field type but wrong", qualify_with_field = "qualifier_field", qualify_with_field_type = java.lang.String.class) + private Map dependsButNotMap = new HashMap(){{ + put("stellar_key", "TO_UPPER('it was not int')"); + }}; + + private Object qualifier_field = new Integer(2); + + @StellarConfiguration + private TestChildConfigObject child = new TestChildConfigObject(1); + + @StellarConfigurationList(name = "list of holders") + private List stellarConfigList = new LinkedList(){{ + add(new TestChildConfigObject(2)); + add(new TestChildConfigObject(3)); + }}; +} From 5516bad34573ef11dc40eb9ed23b241e7d84c75f Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Sun, 3 Dec 2017 19:54:08 -0500 Subject: [PATCH 10/14] fix for exception change --- .../metron/common/configuration/ConfigurationProviderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java index 2c554ebc3e..ff319bd2b3 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java @@ -271,7 +271,7 @@ public void testProvideAllConfiguredNodes() throws Exception { },(s,e) -> { Assert.assertTrue(e.getMessage(),false); }); - } catch (IllegalAccessException e) { + } catch (Exception e) { Assert.assertTrue(e.getMessage(), false); } }); From b3e7cfb8ac76d618d35da97895169b9069e7fba0 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Sun, 3 Dec 2017 20:17:56 -0500 Subject: [PATCH 11/14] fix regression after fixing mapping in prior commit --- .../common/configuration/ConfigurationProviderTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java index ff319bd2b3..7940298a31 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java @@ -52,8 +52,8 @@ public void setUp() throws Exception { expectedStatements = new LinkedList<>(); expectedStatements.add("Apache Metron/PARSER/squid/fieldTransformations/0/Field Mapping/full_hostname"); expectedStatements.add("Apache Metron/PARSER/squid/fieldTransformations/0/Field Mapping/domain_without_subdomains"); - expectedStatements.add("Apache Metron/ENRICHMENT/snort/enrichment/default/foo"); - expectedStatements.add("Apache Metron/ENRICHMENT/snort/enrichment/default/ALL_CAPS"); + expectedStatements.add("Apache Metron/ENRICHMENT/snort/enrichment/default/stellar/config/foo"); + expectedStatements.add("Apache Metron/ENRICHMENT/snort/enrichment/default/stellar/config/ALL_CAPS"); expectedStatements.add("Apache Metron/ENRICHMENT/snort/threatIntel/triageConfig/riskLevelRules/0/rule"); expectedStatements.add("Apache Metron/PROFILER/example2/onlyif"); expectedStatements.add("Apache Metron/PROFILER/example2/init/num_dns"); From 998542c84a731a7b3e84950057ad8befb049a64e Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Fri, 22 Dec 2017 12:44:03 -0500 Subject: [PATCH 12/14] refactored to stellar function in metron management --- metron-platform/metron-management/README.md | 9 + .../management/ValidationFunctions.java | 107 +++++++++++ .../ValidationFunctionsIntegrationTest.java | 180 ++++++++++++++++++ metron-stellar/stellar-common/README.md | 5 - .../stellar/common/shell/StellarExecutor.java | 1 - .../stellar/common/shell/StellarShell.java | 21 -- .../utils/validation/StellarValidator.java | 4 +- .../StellarZookeeperBasedValidator.java | 53 ++---- .../utils/validation/ValidationResult.java | 80 ++++++++ 9 files changed, 398 insertions(+), 62 deletions(-) create mode 100644 metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java create mode 100644 metron-platform/metron-management/src/test/java/org/apache/metron/management/ValidationFunctionsIntegrationTest.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ValidationResult.java diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md index 07a5b1a185..f979ca4e8f 100644 --- a/metron-platform/metron-management/README.md +++ b/metron-platform/metron-management/README.md @@ -24,6 +24,7 @@ project. * [Indexing Functions](#indexing-functions) * [Enrichment Functions](#enrichment-functions) * [Threat Triage Functions](#threat-triage-functions) + * [Validation Functions](#validation-functions) * [Examples](#examples) * [Iterate to Find a Valid Grok Pattern](#iterate-to-find-a-valid-grok-pattern) * [Manage Stellar Field Transformations](#manage-stellar-field-transformations) @@ -41,6 +42,7 @@ The functions are split roughly into a few sections: * Parser functions - Functions surrounding adding, viewing, and removing Parser functions. * Enrichment functions - Functions surrounding adding, viewing and removing Stellar enrichments as well as managing batch size, batch timeout, and index names for the enrichment topology configuration * Threat Triage functions - Functions surrounding adding, viewing and removing threat triage functions. +* Validation functions - Functions to validate stellar rules contained in configurations. ### Grok Functions @@ -276,6 +278,13 @@ The functions are split roughly into a few sections: * aggregatorConfig - Optional config for aggregator * Returns: The String representation of the enrichment config +### Validation Functions +* `VALIDATE_STELLAR_RULE_CONFIGS` + * Description: Attempts to validate deployed Stellar expressions by ensuring they compile. This is useful when expressions may have been valid when deployed, but may have been invalidated by a language change after that + * Input: + * wrap - Optional. The Length of the string to wrap the columns + * Returns: A table of validation results + ## Deployment Instructions * Clusters installed via Ambari Management Pack (default) * Automatically deployed diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java new file mode 100644 index 0000000000..736d83d326 --- /dev/null +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.metron.management; + +import com.jakewharton.fliptables.FlipTable; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.apache.commons.lang3.text.WordUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.validation.StellarZookeeperBasedValidator; +import org.apache.metron.stellar.common.utils.validation.ValidationResult; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; + +/** + * The VALIDATE namespace is used for functions that validate Stellar rule strings. + */ +public class ValidationFunctions { + + private static final String[] headers = new String[]{"PATH", "RULE", "ERROR", "VALID"}; + + @Stellar(namespace = "VALIDATE", name = "STELLAR_RULE_CONFIGS", description = + "Attempts to validate deployed Stellar expressions by ensuring they compile." + + "This is useful when expressions may have been valid when deployed, but may have been" + + "invalidated by a language change after that", params = { + "wrap : Optional. The Length of string to wrap the columns" + }, returns = "A table of validation results") + public static class DiscoverAndValidateStellarRules implements StellarFunction { + + @Override + public Object apply(List args, Context context) { + + int wordWrap = -1; + if(args.size() > 0) { + wordWrap = ConversionUtils.convert(args.get(0), Integer.class); + } + + final int wrapSize = wordWrap; + Optional clientOpt = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + if (!clientOpt.isPresent()) { + throw new IllegalStateException( + "VALIDATE_STELLAR_RULE_CONFIGS requires zookeeper. Please connect to zookeeper."); + } + CuratorFramework client = (CuratorFramework) clientOpt.get(); + + StellarZookeeperBasedValidator validator = new StellarZookeeperBasedValidator(client); + Iterable result = validator.validate(Optional.empty()); + + final ArrayList dataList = new ArrayList<>(); + result.forEach((v) -> { + dataList.add(new String[]{ + toWrappedString(v.getRulePath(),wrapSize), + toWrappedString(v.getRule(),wrapSize), + toWrappedString(v.getError(),wrapSize), + toWrappedString(String.valueOf(v.isValid()),wrapSize) + }); + }); + + if (dataList.isEmpty()) { + return FlipTable.of(headers, new String[][]{}); + } + + String[][] data = new String[dataList.size()][headers.length]; + for (int i = 0; i < dataList.size(); ++i) { + data[i] = dataList.get(i); + } + return FlipTable.of(headers, data); + } + + private static String toWrappedString(Object o, int wrap) { + if (o == null) { + return ""; + } + String s = "" + o; + if (wrap <= 0) { + return s; + } + return WordUtils.wrap(s, wrap); + } + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } + +} diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ValidationFunctionsIntegrationTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ValidationFunctionsIntegrationTest.java new file mode 100644 index 0000000000..534bcb15c0 --- /dev/null +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ValidationFunctionsIntegrationTest.java @@ -0,0 +1,180 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.management; + +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.dsl.Context.Capabilities.ZOOKEEPER_CLIENT; + +import java.util.HashMap; +import java.util.Properties; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.integration.BaseIntegrationTest; +import org.apache.metron.integration.ComponentRunner; +import org.apache.metron.integration.components.ZKServerComponent; +import org.apache.metron.stellar.dsl.Context; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ValidationFunctionsIntegrationTest extends BaseIntegrationTest { + + private static ZKServerComponent zkServerComponent; + private static ComponentRunner runner; + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "foo", + * "parserConfig": { + * "grokPath": "/patterns/foo", + * "patternLabel": "FOO_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR" + * ,"output" : [ "full_hostname", "domain_without_subdomains" ] + * ,"config" : { + * "full_hostname" : "URL_TO_HOST(url)" + * ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + * } + * } + * ] + * } + */ + @Multiline + private static String config; + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "bar", + * "parserConfig": { + * "grokPath": "/patterns/bar", + * "patternLabel": "BAR_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR" + * ,"output" : [ "full_hostname", "domain_without_subdomains" ] + * ,"config" : { + * "full_hostname" : "URL_TO_HOST(url)=" + * ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)=" + * } + * } + * ] + * } + */ + @Multiline + private static String badConfig; + + /** + *╔═════════════════════════════════════════════════════════════════════════════════════════╀══════════════════════════════════════════╀═════════════════════════════════════════════════════╀═══════╗ + *β•‘ PATH β”‚ RULE β”‚ ERROR β”‚ VALID β•‘ + *╠═════════════════════════════════════════════════════════════════════════════════════════β•ͺ══════════════════════════════════════════β•ͺ═════════════════════════════════════════════════════β•ͺ═══════╣ + *β•‘ Apache Metron/PARSER/bar/fieldTransformations/0/Field Mapping/full_hostname β”‚ URL_TO_HOST(url)= β”‚ Syntax error @ 1:16 token recognition error at: '=' β”‚ false β•‘ + *β•Ÿβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β•’ + *β•‘ Apache Metron/PARSER/bar/fieldTransformations/0/Field Mapping/domain_without_subdomains β”‚ DOMAIN_REMOVE_SUBDOMAINS(full_hostname)= β”‚ Syntax error @ 1:39 token recognition error at: '=' β”‚ false β•‘ + *β•Ÿβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β•’ + *β•‘ Apache Metron/PARSER/foo/fieldTransformations/0/Field Mapping/full_hostname β”‚ URL_TO_HOST(url) β”‚ β”‚ true β•‘ + *β•Ÿβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β•’ + *β•‘ Apache Metron/PARSER/foo/fieldTransformations/0/Field Mapping/domain_without_subdomains β”‚ DOMAIN_REMOVE_SUBDOMAINS(full_hostname) β”‚ β”‚ true β•‘ + *β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β• + **/ + @Multiline + private static String expectedOutput; + + + /** + *╔════════════════════════════════════════════════╀══════════════════════════════════════════╀═════════════════════╀═══════╗ + *β•‘ PATH β”‚ RULE β”‚ ERROR β”‚ VALID β•‘ + *╠════════════════════════════════════════════════β•ͺ══════════════════════════════════════════β•ͺ═════════════════════β•ͺ═══════╣ + *β•‘ Apache β”‚ URL_TO_HOST(url)= β”‚ Syntax error @ 1:16 β”‚ false β•‘ + *β•‘ Metron/PARSER/bar/fieldTransformations/0/Field β”‚ β”‚ token recognition β”‚ β•‘ + *β•‘ Mapping/full_hostname β”‚ β”‚ error at: '=' β”‚ β•‘ + *β•Ÿβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β•’ + *β•‘ Apache β”‚ DOMAIN_REMOVE_SUBDOMAINS(full_hostname)= β”‚ Syntax error @ 1:39 β”‚ false β•‘ + *β•‘ Metron/PARSER/bar/fieldTransformations/0/Field β”‚ β”‚ token recognition β”‚ β•‘ + *β•‘ Mapping/domain_without_subdomains β”‚ β”‚ error at: '=' β”‚ β•‘ + *β•Ÿβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β•’ + *β•‘ Apache β”‚ URL_TO_HOST(url) β”‚ β”‚ true β•‘ + *β•‘ Metron/PARSER/foo/fieldTransformations/0/Field β”‚ β”‚ β”‚ β•‘ + *β•‘ Mapping/full_hostname β”‚ β”‚ β”‚ β•‘ + *β•Ÿβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β•’ + *β•‘ Apache β”‚ DOMAIN_REMOVE_SUBDOMAINS(full_hostname) β”‚ β”‚ true β•‘ + *β•‘ Metron/PARSER/foo/fieldTransformations/0/Field β”‚ β”‚ β”‚ β•‘ + *β•‘ Mapping/domain_without_subdomains β”‚ β”‚ β”‚ β•‘ + *β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β• + */ + @Multiline + private static String expectedWrappedOutput; + + @BeforeClass + public static void setupZookeeper() throws Exception { + Properties properties = new Properties(); + zkServerComponent = getZKServerComponent(properties); + zkServerComponent.start(); + } + + @AfterClass + public static void tearDownZookeeper() { + zkServerComponent.stop(); + } + + + @Test + public void test() throws Exception { + try( CuratorFramework client = ConfigurationsUtils.getClient(zkServerComponent.getConnectionString())) { + client.start(); + setupConfiguration(client); + Context context = new Context.Builder().with(ZOOKEEPER_CLIENT, () -> client).build(); + Object out = run("VALIDATE_STELLAR_RULE_CONFIGS()", new HashMap<>(), context); + Assert.assertEquals(expectedOutput, out); + } + + } + @Test + public void testWrapped() throws Exception { + try( CuratorFramework client = ConfigurationsUtils.getClient(zkServerComponent.getConnectionString())) { + client.start(); + setupConfiguration(client); + Context context = new Context.Builder().with(ZOOKEEPER_CLIENT, () -> client).build(); + Object out = run("VALIDATE_STELLAR_RULE_CONFIGS(20)", new HashMap<>(), context); + Assert.assertEquals(expectedWrappedOutput, out); + } + + } + private void setupConfiguration(CuratorFramework client) throws Exception { + + // upload squid configuration to zookeeper + ConfigurationsUtils.writeSensorParserConfigToZookeeper("foo", + config.getBytes(), client); + + ConfigurationsUtils.writeSensorParserConfigToZookeeper("bar", + badConfig.getBytes(), client); + + } + +} \ No newline at end of file diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 36d9ddee54..759a6e7fb2 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -1351,11 +1351,6 @@ IS_EMAIL ret: True if the string is a valid email address and false otherwise. [Stellar]>>> ``` - -#### `%validate_configured_expressions` - -Attempts to validate deployed Stellar expressions by ensuring they compile. This is useful when expressions may have been valid when deployed, but may have been invalidated by a language change after that. - ### Advanced Usage To run the Stellar Shell directly from the Metron source code, run a command like the following. Ensure that Metron has already been built and installed with `mvn clean install -DskipTests`. diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java index 347245009f..eafeeca8ef 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java @@ -208,7 +208,6 @@ private PatriciaTrie initializeIndex() { index.put(StellarShell.MAGIC_GLOBALS, AutoCompleteType.FUNCTION); index.put(StellarShell.MAGIC_DEFINE, AutoCompleteType.FUNCTION); index.put(StellarShell.MAGIC_UNDEFINE, AutoCompleteType.FUNCTION); - index.put(StellarShell.MAGIC_VALIDATE_CONFIGURED, AutoCompleteType.FUNCTION); return new PatriciaTrie<>(index); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index c1d73c3d9c..28d1d9efb5 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -95,7 +95,6 @@ public class StellarShell extends AeshConsoleCallback implements Completion { public static final String MAGIC_GLOBALS = MAGIC_PREFIX + "globals"; public static final String MAGIC_DEFINE = MAGIC_PREFIX + "define"; public static final String MAGIC_UNDEFINE = MAGIC_PREFIX + "undefine"; - public static final String MAGIC_VALIDATE_CONFIGURED = MAGIC_PREFIX + "validate_configured_expressions"; private StellarExecutor executor; @@ -320,8 +319,6 @@ private void handleMagic(String rawExpression) { } else if(MAGIC_UNDEFINE.equals(command)) { handleMagicUndefine(expression); - } else if(MAGIC_VALIDATE_CONFIGURED.equals(command)) { - handleValidateConfigured(); } else { writeLine(ERROR_PROMPT + "undefined magic command: " + rawExpression); } @@ -406,24 +403,6 @@ private void handleMagicUndefine(String[] expression) { } } - /** - * Handle a magic '%validate_configured_expressions'. - * This command will attempt to locate and validate through compilation - * all deployed stellar statements in the system. - */ - private void handleValidateConfigured() { - - Optional client = executor.getClient(); - if (!client.isPresent()) { - writeLine(ERROR_PROMPT + "Zookeeper is required to validate deployed stellar statements!"); - writeLine(ERROR_PROMPT + "Please restart the Stellar Shell with the -z parameter!"); - return; - } - - StellarZookeeperBasedValidator validator = new StellarZookeeperBasedValidator(client.get()); - validator.validate(this::writeLine); - } - /** * Retrieves the GLOBAL_CONFIG, if it exists. If it does not, it creates the GLOBAL_CONFIG * and adds it to the Stellar execution context. diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java index 16bcda0076..a3cf5ba17c 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java @@ -20,9 +20,11 @@ package org.apache.metron.stellar.common.utils.validation; +import java.util.Optional; + public interface StellarValidator { public interface LineWriter { void write(String line); } - void validate(LineWriter writer); + Iterable validate(Optional writer); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java index b8c864380b..759e3f6087 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java @@ -22,17 +22,24 @@ import static org.apache.metron.stellar.common.shell.StellarShell.ERROR_PROMPT; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Optional; import java.util.Set; import org.apache.commons.lang.NullArgumentException; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.stellar.common.StellarProcessor; import org.atteo.classindex.ClassIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class StellarZookeeperBasedValidator implements StellarValidator { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String FAILED_COMPILE = "Failed to compile"; private CuratorFramework client; public StellarZookeeperBasedValidator(CuratorFramework client) throws NullArgumentException { @@ -44,7 +51,7 @@ public StellarZookeeperBasedValidator(CuratorFramework client) throws NullArgume @Override - public void validate(LineWriter writer) { + public Iterable validate(Optional writer) { // discover all the StellarConfigurationProvider Set providerSet = new HashSet<>(); @@ -57,65 +64,43 @@ public void validate(LineWriter writer) { .cast(c.getConstructor().newInstance()); providerSet.add(reporter); } catch (Exception e) { - writer - .write(ERROR_PROMPT + " Provider: " + c.getCanonicalName() + " not valid, skipping"); + LOG.error("Provider: " + c.getCanonicalName() + " not valid, skipping", e); } } } - writer.write(String.format("Discovered %d providers", providerSet.size())); - - writer.write("Visiting all configurations. ThreatTriage rules are checked when loading the " - + "configuration, thus an invalid ThreatTriage rule will fail the entire Enrichment Configuration."); - if (providerSet.size() > 0) { - providerSet.forEach((r) -> writer.write(r.getName() + " ")); - writer.write(""); - } else { - return; - } - + ArrayList results = new ArrayList<>(); providerSet.forEach((r) -> { - writer.write("Requesting configurations from " + r.getName()); try { List holders = r .provideConfigurations(client, (pathName, exception) -> { - writer.write(ERROR_PROMPT + String - .format("Configuration %s is not valid, please review", pathName)); + results.add(new ValidationResult(pathName, null, exception.getMessage(), false)); }); - writer.write(String.format("%s provided %d configurations", r.getName(), holders.size())); holders.forEach((h) -> { try { h.discover(); h.visit((path, statement) -> { - writer.write("=================================================="); - writer.write(String.format("validating %s", path)); try { if (StellarProcessor.compile(statement) == null) { - writer.write(ERROR_PROMPT + String - .format("Statement: %s is not valid, please review", path)); - writer.write(ERROR_PROMPT + String.format("Statement: %s ", statement)); + results.add(new ValidationResult(path, statement, FAILED_COMPILE, false)); + } else { + results.add(new ValidationResult(path, statement, null, true)); } } catch (RuntimeException e) { - writer.write(ERROR_PROMPT + "Error Visiting " + path); - writer.write(e.getMessage()); - writer.write("--"); - writer.write(ERROR_PROMPT + ": " + statement); + results.add(new ValidationResult(path, statement, e.getMessage(), false)); } - writer.write("=================================================="); }, (path, error) -> { - writer.write(ERROR_PROMPT + String - .format("Configuration %s is not valid, please review", path)); + results.add(new ValidationResult(path, null, error.getMessage(), false)); }); } catch (Exception e) { - writer.write(ERROR_PROMPT + " " + e.getMessage()); + LOG.error(e.getMessage(), e); } }); } catch (Exception e) { - writer.write(ERROR_PROMPT + "Error Visiting " + r.getName()); - writer.write(e.getMessage()); + LOG.error(e.getMessage(), e); } }); - writer.write("\nDone validation"); + return results; } } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ValidationResult.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ValidationResult.java new file mode 100644 index 0000000000..b26636bd08 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ValidationResult.java @@ -0,0 +1,80 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +/** + * Result of a validation of a Stellar Rule + */ +public class ValidationResult { + + private String rulePath; + private String rule; + private String error; + private boolean valid; + + public ValidationResult(String rulePath, String rule, String error, boolean valid) { + this.rulePath = rulePath; + this.rule = rule; + this.error = error; + this.valid = valid; + } + + public String getRulePath() { + return rulePath; + } + + public String getRule() { + return rule; + } + + public String getError() { + return error; + } + + public boolean isValid() { + return valid; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + ValidationResult that = (ValidationResult) o; + + return new EqualsBuilder().append(isValid(), that.isValid()).append(getRule(), that.getRule()) + .append(getError(), that.getError()).isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37).append(getRule()).append(getError()).append(isValid()) + .toHashCode(); + } +} From 5e8caaf8af69278cd4543e63d407e550b3558fc1 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Sat, 23 Dec 2017 16:24:34 -0500 Subject: [PATCH 13/14] Some refactors based on feedback wrt field names and unused parameters. Attempt to correct the validator from assuming zookeeper. It is more correct that there will be multple types of validators We may support injection of more than one type of validator, make that more clear. --- .../configuration/FieldTransformer.java | 2 +- ...va => ZookeeperConfigurationProvider.java} | 33 ++++--- .../enrichment/EnrichmentConfig.java | 2 +- ...> ZookeeperConfigurationProviderTest.java} | 19 +++-- .../management/ValidationFunctions.java | 49 +++++++---- .../stellar/common/shell/StellarShell.java | 2 - .../ExpressionConfigurationHolder.java | 10 ++- .../StellarConfigurationProvider.java | 9 +- .../StellarConfiguredStatementContainer.java | 41 +++++++-- .../utils/validation/StellarValidator.java | 6 +- ...StellarZookeeperConfigurationProvider.java | 46 ++++++++++ .../annotations/StellarExpressionMap.java | 10 +-- .../validation/validators/BaseValidator.java | 48 +++++++++++ .../StellarSimpleValidator.java} | 55 +++--------- .../StellarZookeeperBasedValidator.java | 85 +++++++++++++++++++ .../utils/validation/TestConfigObject.java | 6 +- 16 files changed, 311 insertions(+), 112 deletions(-) rename metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/{ConfigurationProvider.java => ZookeeperConfigurationProvider.java} (80%) rename metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/{ConfigurationProviderTest.java => ZookeeperConfigurationProviderTest.java} (92%) create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperConfigurationProvider.java create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java rename metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/{StellarZookeeperBasedValidator.java => validators/StellarSimpleValidator.java} (56%) create mode 100644 metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarZookeeperBasedValidator.java diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java index 9c76237506..e89341dd50 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java @@ -38,7 +38,7 @@ public class FieldTransformer implements Serializable { private FieldTransformation transformation; private String transformationName; @StellarExpressionMap(name = "Field Mapping", - qualify_with_field = "transformation", qualify_with_field_type = StellarTransformation.class) + qualifyWithField = "transformation", qualifyWithFieldType = StellarTransformation.class) private LinkedHashMap config = new LinkedHashMap<>(); private boolean initialized = false; public FieldTransformer() { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ZookeeperConfigurationProvider.java similarity index 80% rename from metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java rename to metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ZookeeperConfigurationProvider.java index a4d8528197..b76ae934f9 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationProvider.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ZookeeperConfigurationProvider.java @@ -19,6 +19,7 @@ import static org.apache.metron.common.configuration.ConfigurationType.PARSER; import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; +import java.lang.invoke.MethodHandles; import java.util.LinkedList; import java.util.List; import org.apache.curator.framework.CuratorFramework; @@ -27,20 +28,23 @@ import org.apache.metron.common.configuration.profiler.ProfilerConfig; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.stellar.common.utils.validation.ExpressionConfigurationHolder; -import org.apache.metron.stellar.common.utils.validation.StellarConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.StellarZookeeperConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer; import org.apache.zookeeper.KeeperException.NoNodeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * {@code ConfigurationProvider} is used to report all of the configured / deployed Stellar statements in + * {@code ZookeeperConfigurationProvider} is used to report all of the configured / deployed Stellar statements in * the system. */ -public class ConfigurationProvider implements StellarConfigurationProvider { - +public class ZookeeperConfigurationProvider implements StellarZookeeperConfigurationProvider { + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** * Default constructor. */ - public ConfigurationProvider() { + public ZookeeperConfigurationProvider() { } @Override @@ -49,9 +53,9 @@ public String getName() { } @Override - public List provideConfigurations(CuratorFramework client, + public List provideContainers(CuratorFramework client, ErrorConsumer errorConsumer) { - List holders = new LinkedList<>(); + List holders = new LinkedList<>(); visitParserConfigs(client, holders, errorConsumer); visitEnrichmentConfigs(client, holders, errorConsumer); visitProfilerConfigs(client, holders, errorConsumer); @@ -59,12 +63,13 @@ public List provideConfigurations(CuratorFramewor } private void visitParserConfigs(CuratorFramework client, - List holders, ErrorConsumer errorConsumer) { + List holders, ErrorConsumer errorConsumer) { List children = null; try { children = client.getChildren().forPath(PARSER.getZookeeperRoot()); - } catch (Exception nne) { + } catch (Exception e) { + LOG.error("Exception getting parser configurations", e); return; } for (String child : children) { @@ -83,12 +88,13 @@ private void visitParserConfigs(CuratorFramework client, @SuppressWarnings("unchecked") private void visitEnrichmentConfigs(CuratorFramework client, - List holders, ErrorConsumer errorConsumer) { + List holders, ErrorConsumer errorConsumer) { List children = null; try { children = client.getChildren().forPath(ENRICHMENT.getZookeeperRoot()); - } catch (Exception nne) { + } catch (Exception e) { + LOG.error("Exception getting enrichment configurations", e); return; } @@ -121,12 +127,13 @@ private void visitEnrichmentConfigs(CuratorFramework client, } private void visitProfilerConfigs(CuratorFramework client, - List holders, ErrorConsumer errorConsumer) { + List holders, ErrorConsumer errorConsumer) { try { byte[] profilerConfigData = null; try { profilerConfigData = client.getData().forPath(PROFILER.getZookeeperRoot()); - } catch (NoNodeException nne) { + } catch (NoNodeException e) { + LOG.error("Exception getting profiler configurations", e); return; } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java index c7bab0ba37..07fda8564f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java @@ -29,7 +29,7 @@ @StellarConfiguration public class EnrichmentConfig { - @StellarExpressionMap(inner_map_keys = {"stellar","config"}) + @StellarExpressionMap(innerMapKeys = {"stellar","config"}) private Map fieldMap = new HashMap<>(); private Map enrichmentConfigs = new HashMap<>(); private Map> fieldToTypeMap = new HashMap<>(); diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ZookeeperConfigurationProviderTest.java similarity index 92% rename from metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java rename to metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ZookeeperConfigurationProviderTest.java index 7940298a31..f1d4fa502e 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationProviderTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ZookeeperConfigurationProviderTest.java @@ -24,12 +24,13 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.test.TestingServer; import org.apache.metron.stellar.common.utils.validation.ExpressionConfigurationHolder; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -public class ConfigurationProviderTest { +public class ZookeeperConfigurationProviderTest { private TestingServer testZkServer; private String zookeeperUrl; @@ -259,9 +260,9 @@ public void testProvideAllConfiguredNodes() throws Exception { zookeeperUrl); ConfigurationsUtils.writeProfilerConfigToZookeeper(profilerConfig.getBytes(), client); - ConfigurationProvider configurationProvider = new ConfigurationProvider(); - List holders = configurationProvider - .provideConfigurations(client, + ZookeeperConfigurationProvider zookeeperConfigurationProvider = new ZookeeperConfigurationProvider(); + List holders = zookeeperConfigurationProvider + .provideContainers(client, (names, error) -> Assert.assertTrue("Should not get errors", false)); holders.forEach((h) -> { try { @@ -276,7 +277,7 @@ public void testProvideAllConfiguredNodes() throws Exception { } }); - holders.forEach((h) -> expected.remove(h.getFullName())); + holders.forEach((h) -> expected.remove((h.getFullName()))); Assert.assertTrue(expected.isEmpty()); foundStatements.removeAll(expectedStatements); @@ -289,8 +290,8 @@ public void testErrorCalledWithBadParserConfig() throws Exception { ConfigurationsUtils .writeToZookeeper(ConfigurationType.PARSER.getZookeeperRoot() + "/" + "squid", badParserConfig.getBytes(), client); - ConfigurationProvider configurationProvider = new ConfigurationProvider(); - configurationProvider.provideConfigurations(client, (names, error) -> { + ZookeeperConfigurationProvider zookeeperConfigurationProvider = new ZookeeperConfigurationProvider(); + zookeeperConfigurationProvider.provideContainers(client, (names, error) -> { Assert.assertTrue("Should get errors", true); }); } @@ -300,8 +301,8 @@ public void testExceptionThrownGetsBadThreatConfig() throws Exception { ConfigurationsUtils .writeSensorEnrichmentConfigToZookeeper("snort", badEnrichmentConfig.getBytes(), zookeeperUrl); - ConfigurationProvider configurationProvider = new ConfigurationProvider(); - configurationProvider.provideConfigurations(client, + ZookeeperConfigurationProvider zookeeperConfigurationProvider = new ZookeeperConfigurationProvider(); + zookeeperConfigurationProvider.provideContainers(client, (names, error) -> Assert.assertTrue("Should not get errors", false)); } } \ No newline at end of file diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java index 736d83d326..7d1ddbd34f 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ValidationFunctions.java @@ -22,7 +22,9 @@ import org.apache.commons.lang3.text.WordUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.stellar.common.utils.ConversionUtils; -import org.apache.metron.stellar.common.utils.validation.StellarZookeeperBasedValidator; +import org.apache.metron.stellar.common.utils.validation.StellarValidator; +import org.apache.metron.stellar.common.utils.validation.validators.StellarSimpleValidator; +import org.apache.metron.stellar.common.utils.validation.validators.StellarZookeeperBasedValidator; import org.apache.metron.stellar.common.utils.validation.ValidationResult; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.Stellar; @@ -39,15 +41,14 @@ public class ValidationFunctions { "Attempts to validate deployed Stellar expressions by ensuring they compile." + "This is useful when expressions may have been valid when deployed, but may have been" + "invalidated by a language change after that", params = { - "wrap : Optional. The Length of string to wrap the columns" - }, returns = "A table of validation results") + "wrap : Optional. The Length of string to wrap the columns"}, returns = "A table of validation results") public static class DiscoverAndValidateStellarRules implements StellarFunction { @Override public Object apply(List args, Context context) { int wordWrap = -1; - if(args.size() > 0) { + if (args.size() > 0) { wordWrap = ConversionUtils.convert(args.get(0), Integer.class); } @@ -58,19 +59,12 @@ public Object apply(List args, Context context) { "VALIDATE_STELLAR_RULE_CONFIGS requires zookeeper. Please connect to zookeeper."); } CuratorFramework client = (CuratorFramework) clientOpt.get(); - - StellarZookeeperBasedValidator validator = new StellarZookeeperBasedValidator(client); - Iterable result = validator.validate(Optional.empty()); - final ArrayList dataList = new ArrayList<>(); - result.forEach((v) -> { - dataList.add(new String[]{ - toWrappedString(v.getRulePath(),wrapSize), - toWrappedString(v.getRule(),wrapSize), - toWrappedString(v.getError(),wrapSize), - toWrappedString(String.valueOf(v.isValid()),wrapSize) - }); - }); + + // validate the zookeeper configurations + // when we support other validators, we will call them as well + validate(new StellarZookeeperBasedValidator(client), dataList, wrapSize); + validate(new StellarSimpleValidator(), dataList, wrapSize); if (dataList.isEmpty()) { return FlipTable.of(headers, new String[][]{}); @@ -93,6 +87,29 @@ private static String toWrappedString(Object o, int wrap) { } return WordUtils.wrap(s, wrap); } + + private void validate(StellarValidator validator, ArrayList dataList, int wrapSize) { + Iterable result = validator.validate(); + + result.forEach((v) -> { + dataList.add(new String[]{toWrappedString(v.getRulePath(), wrapSize), + toWrappedString(v.getRule(), wrapSize), toWrappedString(v.getError(), wrapSize), + toWrappedString(String.valueOf(v.isValid()), wrapSize)}); + }); + + validator = new StellarSimpleValidator(); + result = validator.validate(); + + if (dataList.isEmpty()) { + return; + } + + String[][] data = new String[dataList.size()][headers.length]; + for (int i = 0; i < dataList.size(); ++i) { + data[i] = dataList.get(i); + } + } + @Override public void initialize(Context context) { diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index 28d1d9efb5..810b2beb06 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -26,11 +26,9 @@ import com.google.common.collect.Lists; import org.apache.commons.cli.*; import org.apache.commons.lang3.StringUtils; -import org.apache.curator.framework.CuratorFramework; import org.apache.log4j.PropertyConfigurator; import org.apache.metron.stellar.common.StellarAssignment; import org.apache.metron.stellar.common.utils.JSONUtils; -import org.apache.metron.stellar.common.utils.validation.StellarZookeeperBasedValidator; import org.apache.metron.stellar.dsl.StellarFunctionInfo; import org.jboss.aesh.complete.CompleteOperation; import org.jboss.aesh.complete.Completion; diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java index c7f7200178..64ef482d42 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/ExpressionConfigurationHolder.java @@ -100,6 +100,7 @@ public Object getHolderObject() { * * @return String value of the name */ + @Override public String getName() { return name; } @@ -119,6 +120,7 @@ public String getParentName() { * * @return String value of the name */ + @Override public String getFullName() { return fullName; } @@ -191,9 +193,9 @@ private void visitMaps(StatementVisitor visitor, ErrorConsumer errorConsumer) { // for example, if StellarFoo.class.isAssignableFrom(this.foo.getClass()) then // bar is a StellarExpressionMap if (!StringUtils - .isEmpty(f.getAnnotation(StellarExpressionMap.class).qualify_with_field())) { - String fieldName = f.getAnnotation(StellarExpressionMap.class).qualify_with_field(); - Class type = f.getAnnotation(StellarExpressionMap.class).qualify_with_field_type(); + .isEmpty(f.getAnnotation(StellarExpressionMap.class).qualifyWithField())) { + String fieldName = f.getAnnotation(StellarExpressionMap.class).qualifyWithField(); + Class type = f.getAnnotation(StellarExpressionMap.class).qualifyWithFieldType(); Object theObject = FieldUtils.readField(holderObject, fieldName, true); if (theObject == null) { errorConsumer.consume(thisFullName, @@ -208,7 +210,7 @@ private void visitMaps(StatementVisitor visitor, ErrorConsumer errorConsumer) { Map map = (Map) FieldUtils.readField(f, holderObject, true); // some maps actually nest the config, so check and dig to get the real map - String[] innerKeys = f.getAnnotation(StellarExpressionMap.class).inner_map_keys(); + String[] innerKeys = f.getAnnotation(StellarExpressionMap.class).innerMapKeys(); if (innerKeys.length != 0) { for (String key : innerKeys) { if (StringUtils.isEmpty(key)) { diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java index 645d7e43bf..b182d54d8b 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfigurationProvider.java @@ -19,7 +19,6 @@ package org.apache.metron.stellar.common.utils.validation; import java.util.List; -import org.apache.curator.framework.CuratorFramework; import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer; import org.atteo.classindex.IndexSubclasses; @@ -29,7 +28,6 @@ */ @IndexSubclasses public interface StellarConfigurationProvider { - /** * The Name of this reporter. * @return String @@ -37,10 +35,9 @@ public interface StellarConfigurationProvider { String getName(); /** - * Returns a list of all known ExpressionConfigurationHolders. - * @param client The Zookeeper client - * @return List of ExpressionConfigurationHolder + * Returns a list of all known StellarConfiguredStatementContainer. + * @return List of StellarConfiguredStatementContainer */ - List provideConfigurations(CuratorFramework client, ErrorConsumer errorConsumer); + List provideContainers(ErrorConsumer errorConsumer); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java index 2e5bd4ed40..4fe7065b64 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarConfiguredStatementContainer.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,19 +20,50 @@ /** * StellarConfiguredStatementProviders are used provide stellar statements - * and the context around those statements to the caller + * and the context around those statements to the caller. + * + * The usage pattern is: + * + * stellarConfiguredStatementContainer.discover(); + * stellarConfiguredStatementContiner.visit((p,s)->{}, (p,e) -> {} ); + * */ public interface StellarConfiguredStatementContainer { - public interface StatementVisitor { + interface StatementVisitor { void visit(String path, String statement); } - public interface ErrorConsumer { + interface ErrorConsumer { void consume(String path, Exception e); } - void visit(StatementVisitor visitor, ErrorConsumer errorConsumer) throws Exception; + /** + * Returns a name for this container. + * @return String + */ + String getName(); + + /** + * Returns a full name for this container. + * This would include pathing information if applicable. + * @return String + */ + String getFullName(); + + /** + * Discover is called to allow the Container to discover it's configurations. + * @throws Exception if there is an error + */ void discover() throws Exception; + /** + * Visit each configuration within this container. + * @param visitor StatementVisitor callback + * @param errorConsumer ErrorConsumer callback + * @throws Exception if there is an error + */ + void visit(StatementVisitor visitor, ErrorConsumer errorConsumer) throws Exception; + + } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java index a3cf5ba17c..69eaba29d0 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarValidator.java @@ -20,11 +20,7 @@ package org.apache.metron.stellar.common.utils.validation; -import java.util.Optional; public interface StellarValidator { - public interface LineWriter { - void write(String line); - } - Iterable validate(Optional writer); + Iterable validate(); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperConfigurationProvider.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperConfigurationProvider.java new file mode 100644 index 0000000000..cc3244e946 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperConfigurationProvider.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils.validation; + +import java.util.List; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer; +import org.atteo.classindex.IndexSubclasses; + +/** + * {@code StellarZookeeperConfigurationProvider} are used provide Stellar statements + * and the context around those statements to the caller. + */ +@IndexSubclasses +public interface StellarZookeeperConfigurationProvider { + + /** + * The Name of this reporter. + * @return String + */ + String getName(); + + /** + * Returns a list of all known StellarConfiguredStatementContainer. + * @param client The Zookeeper client + * @return List of StellarConfiguredStatementContainer + */ + List provideContainers(CuratorFramework client, ErrorConsumer errorConsumer); + +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java index d914b76e14..01e898bea7 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/annotations/StellarExpressionMap.java @@ -52,21 +52,21 @@ * A map may be a StellarExpressionMap based on the type * of another field, this is that field's name. * - * {@code} qualify_with_field_type} is the type of that field + * {@code} qualifyWithFieldType} is the type of that field * * @return Field Name or empty String */ - String qualify_with_field() default ""; + String qualifyWithField() default ""; /** * A map may be a StellarExpressionMap based on the type * of another field, this is that type of that field. * - * {@code qualify_with_field} is the name of the field. + * {@code qualifyWithField} is the name of the field. * * @return Class */ - Class qualify_with_field_type() default Error.class; + Class qualifyWithFieldType() default Error.class; /** @@ -78,6 +78,6 @@ * * @return String key */ - String[] inner_map_keys() default {}; + String[] innerMapKeys() default {}; } \ No newline at end of file diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java new file mode 100644 index 0000000000..cf3e131ff5 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java @@ -0,0 +1,48 @@ +package org.apache.metron.stellar.common.utils.validation.validators; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; +import org.apache.metron.stellar.common.utils.validation.StellarValidator; +import org.apache.metron.stellar.common.utils.validation.ValidationResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BaseValidator implements StellarValidator { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String FAILED_COMPILE = "Failed to compile"; + + public BaseValidator() { + } + + @Override + public abstract Iterable validate(); + + protected List handleContainers( + List containers) { + ArrayList results = new ArrayList<>(); + containers.forEach((container) -> { + try { + container.discover(); + container.visit((path, statement) -> { + try { + if (StellarProcessor.compile(statement) == null) { + results.add(new ValidationResult(path, statement, FAILED_COMPILE, false)); + } else { + results.add(new ValidationResult(path, statement, null, true)); + } + } catch (RuntimeException e) { + results.add(new ValidationResult(path, statement, e.getMessage(), false)); + } + }, (path, error) -> { + results.add(new ValidationResult(path, null, error.getMessage(), false)); + }); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + }); + return results; + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarSimpleValidator.java similarity index 56% rename from metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java rename to metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarSimpleValidator.java index 759e3f6087..c0fba30661 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/StellarZookeeperBasedValidator.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarSimpleValidator.java @@ -18,44 +18,35 @@ * */ -package org.apache.metron.stellar.common.utils.validation; - -import static org.apache.metron.stellar.common.shell.StellarShell.ERROR_PROMPT; +package org.apache.metron.stellar.common.utils.validation.validators; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; -import java.util.Optional; import java.util.Set; -import org.apache.commons.lang.NullArgumentException; -import org.apache.curator.framework.CuratorFramework; import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.utils.validation.StellarConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; +import org.apache.metron.stellar.common.utils.validation.StellarValidator; +import org.apache.metron.stellar.common.utils.validation.StellarZookeeperConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.ValidationResult; import org.atteo.classindex.ClassIndex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StellarZookeeperBasedValidator implements StellarValidator { - +public class StellarSimpleValidator extends BaseValidator { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String FAILED_COMPILE = "Failed to compile"; - private CuratorFramework client; - - public StellarZookeeperBasedValidator(CuratorFramework client) throws NullArgumentException { - if (client == null) { - throw new NullArgumentException("client"); - } - this.client = client; - } + public StellarSimpleValidator(){} @Override - public Iterable validate(Optional writer) { + public Iterable validate() { // discover all the StellarConfigurationProvider Set providerSet = new HashSet<>(); - for (Class c : ClassIndex.getSubclasses(StellarConfigurationProvider.class, + for (Class c : ClassIndex.getSubclasses(StellarZookeeperConfigurationProvider.class, Thread.currentThread().getContextClassLoader())) { boolean isAssignable = StellarConfigurationProvider.class.isAssignableFrom(c); if (isAssignable) { @@ -72,31 +63,11 @@ public Iterable validate(Optional writer) { ArrayList results = new ArrayList<>(); providerSet.forEach((r) -> { try { - List holders = r - .provideConfigurations(client, (pathName, exception) -> { + List containers = r + .provideContainers((pathName, exception) -> { results.add(new ValidationResult(pathName, null, exception.getMessage(), false)); }); - - holders.forEach((h) -> { - try { - h.discover(); - h.visit((path, statement) -> { - try { - if (StellarProcessor.compile(statement) == null) { - results.add(new ValidationResult(path, statement, FAILED_COMPILE, false)); - } else { - results.add(new ValidationResult(path, statement, null, true)); - } - } catch (RuntimeException e) { - results.add(new ValidationResult(path, statement, e.getMessage(), false)); - } - }, (path, error) -> { - results.add(new ValidationResult(path, null, error.getMessage(), false)); - }); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - }); + results.addAll(handleContainers(containers)); } catch (Exception e) { LOG.error(e.getMessage(), e); } diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarZookeeperBasedValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarZookeeperBasedValidator.java new file mode 100644 index 0000000000..d564e47208 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/StellarZookeeperBasedValidator.java @@ -0,0 +1,85 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.utils.validation.validators; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.commons.lang.NullArgumentException; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer; +import org.apache.metron.stellar.common.utils.validation.StellarValidator; +import org.apache.metron.stellar.common.utils.validation.StellarZookeeperConfigurationProvider; +import org.apache.metron.stellar.common.utils.validation.ValidationResult; +import org.atteo.classindex.ClassIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StellarZookeeperBasedValidator extends BaseValidator { + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private CuratorFramework client; + + public StellarZookeeperBasedValidator(CuratorFramework client) throws NullArgumentException { + if (client == null) { + throw new NullArgumentException("client"); + } + this.client = client; + } + + + @Override + public Iterable validate() { + // discover all the StellarZookeeperConfigurationProvider + Set providerSet = new HashSet<>(); + + for (Class c : ClassIndex.getSubclasses(StellarZookeeperConfigurationProvider.class, + Thread.currentThread().getContextClassLoader())) { + boolean isAssignable = StellarZookeeperConfigurationProvider.class.isAssignableFrom(c); + if (isAssignable) { + try { + StellarZookeeperConfigurationProvider reporter = StellarZookeeperConfigurationProvider.class + .cast(c.getConstructor().newInstance()); + providerSet.add(reporter); + } catch (Exception e) { + LOG.error("Provider: " + c.getCanonicalName() + " not valid, skipping", e); + } + } + } + + ArrayList results = new ArrayList<>(); + providerSet.forEach((r) -> { + try { + List containers = r + .provideContainers(client, (pathName, exception) -> { + results.add(new ValidationResult(pathName, null, exception.getMessage(), false)); + }); + results.addAll(handleContainers(containers)); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + }); + return results; + } +} diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java index 877aa62036..004821ddba 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/validation/TestConfigObject.java @@ -52,7 +52,7 @@ public class TestConfigObject { // a map with the 'real' map burried 2 layers in - @StellarExpressionMap(name = "map two deep", inner_map_keys = {"level1", "level2"}) + @StellarExpressionMap(name = "map two deep", innerMapKeys = {"level1", "level2"}) private Map mapTwoDeep = new HashMap(){{ put("level1",new HashMap(){{ put("level2",new HashMap(){{ @@ -61,12 +61,12 @@ public class TestConfigObject { }}); }}; - @StellarExpressionMap(name = "map depending on field type", qualify_with_field = "qualifier_field", qualify_with_field_type = java.lang.Integer.class) + @StellarExpressionMap(name = "map depending on field type", qualifyWithField = "qualifier_field", qualifyWithFieldType = java.lang.Integer.class) private Map dependsMap = new HashMap(){{ put("stellar_key", "TO_UPPER('it was int')"); }}; - @StellarExpressionMap(name = "map depending on field type but wrong", qualify_with_field = "qualifier_field", qualify_with_field_type = java.lang.String.class) + @StellarExpressionMap(name = "map depending on field type but wrong", qualifyWithField = "qualifier_field", qualifyWithFieldType = java.lang.String.class) private Map dependsButNotMap = new HashMap(){{ put("stellar_key", "TO_UPPER('it was not int')"); }}; From 28745f5281897f15fda60026aae1ac63448feb57 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Sat, 23 Dec 2017 17:47:31 -0500 Subject: [PATCH 14/14] rat check fix --- .../validation/validators/BaseValidator.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java index cf3e131ff5..cddddc0033 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/validation/validators/BaseValidator.java @@ -1,3 +1,23 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.metron.stellar.common.utils.validation.validators; import java.lang.invoke.MethodHandles;