METRON-1339 Stellar Shell functionality to verify stored stellar statements

dependencies_with_url.csv

@@ -177,6 +177,8 @@ commons-lang:commons-lang:jar:2.4:compile,ASLv2,http://commons.apache.org/lang/
 commons-lang:commons-lang:jar:2.5:compile,ASLv2,http://commons.apache.org/lang/
 commons-lang:commons-lang:jar:2.6:compile,ASLv2,http://commons.apache.org/lang/
 commons-lang:commons-lang:jar:2.6:provided,ASLv2,http://commons.apache.org/lang/
+commons-lang:commons-lang:jar:3.7:compile,ASLv2,http://commons.apache.org/lang/
+commons-lang:commons-lang:jar:3.7:provided,ASLv2,http://commons.apache.org/lang/
 commons-logging:commons-logging:jar:1.1.1:compile,ASLv2,http://commons.apache.org/logging
 commons-logging:commons-logging:jar:1.1.3:compile,ASLv2,http://commons.apache.org/proper/commons-logging/
 commons-logging:commons-logging:jar:1.2:compile,ASLv2,http://commons.apache.org/proper/commons-logging/

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java

@@ -20,6 +20,9 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableList;
+import org.apache.metron.common.field.transformation.StellarTransformation;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.common.field.transformation.FieldTransformation;
 import org.apache.metron.common.field.transformation.FieldTransformations;
@@ -28,11 +31,14 @@
 import java.io.Serializable;
 import java.util.*;
 
+@StellarConfigurationList(name = "FieldTransformer")
 public class FieldTransformer implements Serializable {
   private List<String> input = new ArrayList<>();
   private List<String> output;
   private FieldTransformation transformation;
   private String transformationName;
+  @StellarExpressionMap(name = "Field Mapping",
+  qualifyWithField = "transformation", qualifyWithFieldType = StellarTransformation.class)
   private LinkedHashMap<String, Object> config = new LinkedHashMap<>();
   private boolean initialized = false;
   public FieldTransformer() {

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java

@@ -26,7 +26,10 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList;
 
+@StellarConfiguration
 public class SensorParserConfig implements Serializable {
 
   private String parserClassName;
@@ -229,6 +232,7 @@ public void setWriterClassName(String classNames) {
     this.writerClassName = classNames;
   }
   private Map<String, Object> parserConfig = new HashMap<>();
+  @StellarConfigurationList(name = "fieldTransformations")
   private List<FieldTransformer> fieldTransformations = new ArrayList<>();
 
   public List<FieldTransformer> getFieldTransformations() {

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ZookeeperConfigurationProvider.java

@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements.  See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership.  The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License.  You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.metron.common.configuration;
+
+import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT;
+import static org.apache.metron.common.configuration.ConfigurationType.PARSER;
+import static org.apache.metron.common.configuration.ConfigurationType.PROFILER;
+
+import java.lang.invoke.MethodHandles;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.stellar.common.utils.validation.ExpressionConfigurationHolder;
+import org.apache.metron.stellar.common.utils.validation.StellarZookeeperConfigurationProvider;
+import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer;
+import org.apache.metron.stellar.common.utils.validation.StellarConfiguredStatementContainer.ErrorConsumer;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@code ZookeeperConfigurationProvider} is used to report all of the configured / deployed Stellar statements in
+ * the system.
+ */
+public class ZookeeperConfigurationProvider implements StellarZookeeperConfigurationProvider {
+  protected static final Logger LOG =  LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  /**
+   * Default constructor.
+   */
+  public ZookeeperConfigurationProvider() {
+  }
+
+  @Override
+  public String getName() {
+    return "Apache Metron";
+  }
+
+  @Override
+  public List<StellarConfiguredStatementContainer> provideContainers(CuratorFramework client,
+      ErrorConsumer errorConsumer) {
+    List<StellarConfiguredStatementContainer> holders = new LinkedList<>();
+    visitParserConfigs(client, holders, errorConsumer);
+    visitEnrichmentConfigs(client, holders, errorConsumer);
+    visitProfilerConfigs(client, holders, errorConsumer);
+    return holders;
+  }
+
+  private void visitParserConfigs(CuratorFramework client,
+      List<StellarConfiguredStatementContainer> holders, ErrorConsumer errorConsumer) {
+    List<String> children = null;
+
+    try {
+      children = client.getChildren().forPath(PARSER.getZookeeperRoot());
+    } catch (Exception e) {
+      LOG.error("Exception getting parser configurations", e);
+      return;
+    }
+    for (String child : children) {
+      try {
+        byte[] data = client.getData().forPath(PARSER.getZookeeperRoot() + "/" + child);
+        SensorParserConfig parserConfig = SensorParserConfig.fromBytes(data);
+        ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder(
+            String.format("%s/%s", getName(), PARSER.toString()), parserConfig.getSensorTopic(),
+            parserConfig);
+        holders.add(holder);
+      } catch (Exception e) {
+        errorConsumer.consume(String.format("%s/%s/%s", getName(), PARSER.toString(), child), e);
+      }
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private void visitEnrichmentConfigs(CuratorFramework client,
+      List<StellarConfiguredStatementContainer> holders, ErrorConsumer errorConsumer) {
+    List<String> children = null;
+
+    try {
+      children = client.getChildren().forPath(ENRICHMENT.getZookeeperRoot());
+    } catch (Exception e) {
+      LOG.error("Exception getting enrichment configurations", e);
+      return;
+    }
+
+    for (String child : children) {
+
+      try {
+        byte[] data = client.getData().forPath(ENRICHMENT.getZookeeperRoot() + "/" + child);
+        // Certain parts of the SensorEnrichmentConfig do Stellar Verification on their
+        // own as part of deserialization, where the bean spec will call the setter, which has
+        // been wired with stellar verification calls.  There is no avoiding this.
+        //
+        // In cases where those parts of the config are in fact the parts that have invalid
+        // Stellar statements, we will fail during the JSON load before we get to ANY config
+        // contained in the SensorEnrichmentConfig.
+        //
+        // I have left the code to properly check all the configuration parts for completeness
+        // on the reporting side ( the report initiator may want to list successful evals), even
+        // though they can be executed, then they will never fail.
+        final SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig
+            .fromBytes(data);
+        ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder(
+            String.format("%s/%s", getName(), ENRICHMENT.toString()), child,
+            sensorEnrichmentConfig);
+        holders.add(holder);
+      } catch (Exception e) {
+        errorConsumer
+            .consume(String.format("%s/%s/%s", getName(), ENRICHMENT.toString(), child), e);
+      }
+    }
+  }
+
+  private void visitProfilerConfigs(CuratorFramework client,
+      List<StellarConfiguredStatementContainer> holders, ErrorConsumer errorConsumer) {
+    try {
+      byte[] profilerConfigData = null;
+      try {
+        profilerConfigData = client.getData().forPath(PROFILER.getZookeeperRoot());
+      } catch (NoNodeException e) {
+        LOG.error("Exception getting profiler configurations", e);
+        return;
+      }
+
+      ProfilerConfig profilerConfig = JSONUtils.INSTANCE
+          .load(new String(profilerConfigData), ProfilerConfig.class);
+      profilerConfig.getProfiles().forEach((ProfileConfig pc) -> {
+        ExpressionConfigurationHolder holder = new ExpressionConfigurationHolder(
+            String.format("%s/%s", getName(), PROFILER.toString()), pc.getProfile(), pc);
+        holders.add(holder);
+      });
+    } catch (Exception e) {
+      errorConsumer.consume(String.format("%s/%s", getName(), PROFILER.toString()), e);
+    }
+  }
+}

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java

@@ -24,8 +24,12 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap;
 
+@StellarConfiguration
 public class EnrichmentConfig {
+  @StellarExpressionMap(innerMapKeys = {"stellar","config"})
   private Map<String, Object> fieldMap = new HashMap<>();
   private Map<String, ConfigHandler> enrichmentConfigs = new HashMap<>();
   private Map<String, List<String>> fieldToTypeMap = new HashMap<>();

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java

@@ -24,10 +24,14 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration;
 
+@StellarConfiguration
 public class SensorEnrichmentConfig {
 
+  @StellarConfiguration
   private EnrichmentConfig enrichment = new EnrichmentConfig();
+  @StellarConfiguration
   private ThreatIntelConfig threatIntel = new ThreatIntelConfig();
   private Map<String, Object> configuration = new HashMap<>();
 

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java

@@ -17,6 +17,9 @@
  */
 package org.apache.metron.common.configuration.enrichment.threatintel;
 
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField;
+
 /**
  * This class represents a rule that is used to triage threats.
  *
@@ -29,6 +32,7 @@
  * Tuning the threat triage process involves creating one or more rules, adjusting
  * the score of each rule, and changing the way that each rule's score is aggregated.
  */
+@StellarConfiguration
 public class RiskLevelRule {
 
   /**
@@ -45,6 +49,7 @@ public class RiskLevelRule {
    * A predicate, in the form of a Stellar expression, that determines whether
    * the rule is applied to an alert or not.  This field is required.
    */
+  @StellarExpressionField(name = "rule")
   String rule;
 
   /**
@@ -60,6 +65,7 @@ public class RiskLevelRule {
    * This is expected to be a valid Stellar expression and can refer to any of the
    * fields within the message itself.
    */
+  @StellarExpressionField(name = "reason")
   String reason;
 
   public String getName() {

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java

@@ -19,8 +19,11 @@
 package org.apache.metron.common.configuration.enrichment.threatintel;
 
 import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration;
 
+@StellarConfiguration
 public class ThreatIntelConfig extends EnrichmentConfig {
+  @StellarConfiguration
   private ThreatTriageConfig triageConfig = new ThreatTriageConfig();
 
   public ThreatTriageConfig getTriageConfig() {

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java

@@ -29,9 +29,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfigurationList;
 
 public class ThreatTriageConfig {
 
+  @StellarConfigurationList(name = "riskLevelRules")
   private List<RiskLevelRule> riskLevelRules = new ArrayList<>();
   private Aggregators aggregator = Aggregators.MAX;
   private Map<String, Object> aggregationConfig = new HashMap<>();

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java

@@ -26,10 +26,15 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionList;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionMap;
 
 /**
  * The definition of a single Profile.
  */
+@StellarConfiguration
 public class ProfileConfig implements Serializable {
 
   /**
@@ -52,6 +57,7 @@ public class ProfileConfig implements Serializable {
    * is only applied to a profile if this condition is true. This allows a profile
    * to filter the messages that it receives.
    */
+  @StellarExpressionField(name = "onlyif")
   private String onlyif = "true";
 
   /**
@@ -61,26 +67,30 @@ public class ProfileConfig implements Serializable {
    * period the expression is executed once and stored in a variable with the given
    * name.
    */
+  @StellarExpressionMap(name = "init")
   private Map<String, String> init = new HashMap<>();
 
   /**
    * A set of expressions that is executed when a message is applied to the profile.
    * A map is expected where the key is the variable name and the value is a Stellar
    * expression.  The map can include 0 or more variables/expressions.
    */
+  @StellarExpressionMap(name = "update")
   private Map<String, String> update = new HashMap<>();
 
   /**
    * A list of Stellar expressions that is executed in order and used to group the
    * resulting profile data.
    */
+  @StellarExpressionList(name = "groupBy")
   private List<String> groupBy = new ArrayList<>();
 
   /**
    * Stellar expression(s) that are executed when the window period expires.  The
    * expression(s) are expected to in some way summarize the messages that were applied
    * to the profile over the window period.
    */
+  @StellarConfiguration
   private ProfileResult result;
 
   /**

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java

@@ -19,17 +19,20 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration;
 
 /**
  * Defines the 'result' field of a Profile definition.
  */
+@StellarConfiguration
 public class ProfileResult {
 
   /**
    * A Stellar expression that is executed to produce
    * a measurement that is persisted in the profile store.
    */
   @JsonProperty("profile")
+  @StellarConfiguration
   private ProfileResultExpressions profileExpressions;
 
   /**
@@ -38,6 +41,7 @@ public class ProfileResult {
    * triage.
    */
   @JsonProperty("triage")
+  @StellarConfiguration
   private ProfileTriageExpressions triageExpressions;
 
   @JsonCreator

metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java

@@ -19,14 +19,18 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarConfiguration;
+import org.apache.metron.stellar.common.utils.validation.annotations.StellarExpressionField;
 
 /**
  * A Stellar expression that is executed to produce a single
  * measurement that is persisted within the profile store.
  */
+@StellarConfiguration
 public class ProfileResultExpressions {
 
   @JsonIgnore
+  @StellarExpressionField(name = "expression")
   private String expression;
 
   @JsonCreator