From 56be7560dd42fefadfb0d88691b76993943da449 Mon Sep 17 00:00:00 2001 From: rmerriman Date: Wed, 12 Oct 2016 13:46:45 -0500 Subject: [PATCH 1/7] Grok patterns are now read from zookeeper parser config property "grokPattern" --- .../main/config/zookeeper/parsers/squid.json | 2 +- .../config/zookeeper/parsers/websphere.json | 2 +- .../main/config/zookeeper/parsers/yaf.json | 2 +- .../org/apache/metron/parsers/GrokParser.java | 59 +++--- .../metron/parsers/asa/GrokAsaParser.java | 3 +- .../metron/parsers/bolt/ParserBolt.java | 2 +- .../metron/parsers/bro/BasicBroParser.java | 3 +- .../apache/metron/parsers/csv/CSVParser.java | 3 +- .../parsers/fireeye/BasicFireEyeParser.java | 3 +- .../parsers/interfaces/MessageParser.java | 10 +- .../metron/parsers/ise/BasicIseParser.java | 3 +- .../metron/parsers/json/JSONMapParser.java | 3 +- .../parsers/lancope/BasicLancopeParser.java | 3 +- .../parsers/logstash/BasicLogstashParser.java | 3 +- .../paloalto/BasicPaloAltoFirewallParser.java | 3 +- .../parsers/snort/BasicSnortParser.java | 3 +- .../sourcefire/BasicSourcefireParser.java | 3 +- .../apache/metron/parsers/GrokParserTest.java | 9 +- .../metron/parsers/MessageParserTest.java | 13 +- .../metron/parsers/SampleGrokParserTest.java | 41 +++- .../metron/parsers/SnortParserTest.java | 7 +- .../metron/parsers/SquidParserTest.java | 4 +- .../apache/metron/parsers/YafParserTest.java | 5 +- .../metron/parsers/asa/GrokAsaParserTest.java | 5 +- .../metron/parsers/bolt/ParserBoltTest.java | 199 +++++++++--------- .../parsers/bro/BasicBroParserTest.java | 27 +-- .../metron/parsers/bro/BroParserTest.java | 9 +- .../metron/parsers/csv/CSVParserTest.java | 12 +- .../fireeye/BasicFireEyeParserTest.java | 5 +- .../parsers/ise/BasicIseParserTest.java | 23 +- .../parsers/json/JSONMapParserTest.java | 15 +- .../lancope/BasicLancopeParserTest.java | 33 +-- .../BasicPaloAltoFirewallParserTest.java | 5 +- .../sourcefire/BasicSourcefireParserTest.java | 5 +- .../websphere/GrokWebSphereParserTest.java | 60 +++++- 35 files changed, 351 insertions(+), 236 deletions(-) diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json index e44c4c28ea..beec36cb16 100644 --- a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json +++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json @@ -2,7 +2,7 @@ "parserClassName": "org.apache.metron.parsers.GrokParser", "sensorTopic": "squid", "parserConfig": { - "grokPath": "/patterns/squid", + "grokPattern": "SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?", "patternLabel": "SQUID_DELIMITED", "timestampField": "timestamp" }, diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json index 0f2c901415..2a84f4403b 100644 --- a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json +++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json @@ -3,7 +3,7 @@ "sensorTopic":"websphere", "parserConfig": { - "grokPath":"/patterns/websphere", + "grokPattern":"# Days - two digit number is used\nDAY \\d{1,2}\n# Time - two digit hour, minute, and second\nTIME \\d{2}:\\d{2}:\\d{2}\n# Timestamp - month, day, and time\nTIMESTAMP %{MONTH:UNWANTED}\\s+%{DAY:UNWANTED} %{TIME:UNWANTED}\n# Generic word field\nWORD \\w+\n# Priority\nPRIORITY \\d+\n# Log start - the first part of the log line\nLOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname}\n# Security domain\nSECURITY_DOMAIN [%{WORD:security_domain}]\n# Log middle - the middle part of the log line\nLOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]\n# Define IP address formats\nIPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?\nIPV4 (?, Serializable { @@ -47,16 +46,16 @@ public class GrokParser implements MessageParser, Serializable { protected static final Logger LOG = LoggerFactory.getLogger(GrokParser.class); protected transient Grok grok; - protected String grokPath; + protected String grokPattern; protected String patternLabel; protected List timeFields = new ArrayList<>(); protected String timestampField; protected SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z"); - protected String patternsCommonDir = "/patterns/common"; + protected String patternsCommonPath = "/patterns/common"; @Override public void configure(Map parserConfig) { - this.grokPath = (String) parserConfig.get("grokPath"); + this.grokPattern = (String) parserConfig.get("grokPattern"); this.patternLabel = (String) parserConfig.get("patternLabel"); this.timestampField = (String) parserConfig.get("timestampField"); List timeFieldsParam = (List) parserConfig.get("timeFields"); @@ -77,41 +76,30 @@ public void configure(Map parserConfig) { } } - public InputStream openInputStream(String streamName) throws IOException { - FileSystem fs = FileSystem.get(new Configuration()); - Path path = new Path(streamName); - if(fs.exists(path)) { - return fs.open(path); - } else { - return getClass().getResourceAsStream(streamName); - } - } - @Override public void init() { grok = new Grok(); try { - InputStream commonInputStream = openInputStream(patternsCommonDir); + InputStream commonInputStream = getClass().getResourceAsStream(patternsCommonPath); if (LOG.isDebugEnabled()) { - LOG.debug("Grok parser loading common patterns from: " + patternsCommonDir); + LOG.debug("Grok parser loading common patterns from: " + patternsCommonPath); } if (commonInputStream == null) { throw new RuntimeException( - "Unable to initialize grok parser: Unable to load " + patternsCommonDir + " from either classpath or HDFS"); + "Unable to initialize grok parser: Unable to load " + patternsCommonPath + " from either classpath or HDFS"); } grok.addPatternFromReader(new InputStreamReader(commonInputStream)); + if (LOG.isDebugEnabled()) { - LOG.debug("Loading parser-specific patterns from: " + grokPath); + LOG.debug("Loading parser-specific patterns: " + grokPattern); } - InputStream patterInputStream = openInputStream(grokPath); - if (patterInputStream == null) { - throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + grokPath - + " from either classpath or HDFS"); + if (grokPattern == null) { + throw new RuntimeException("Unable to initialize grok parser: grokPattern config property is empty"); } - grok.addPatternFromReader(new InputStreamReader(patterInputStream)); + grok.addPatternFromReader(new InputStreamReader(new ByteArrayInputStream(grokPattern.getBytes()))); if (LOG.isDebugEnabled()) { LOG.debug("Grok parser set the following grok expression: " + grok.getNamedRegexCollectionById(patternLabel)); @@ -132,8 +120,9 @@ public void init() { @SuppressWarnings("unchecked") @Override - public List parse(byte[] rawMessage) { - if (grok == null) { + public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { + if (grok == null || isGrokPatternUpdated(sensorParserConfig) || isPatternLabelUpdated(sensorParserConfig)) { + configure(sensorParserConfig.getParserConfig()); init(); } List messages = new ArrayList<>(); @@ -150,8 +139,8 @@ public List parse(byte[] rawMessage) { if (message.size() == 0) throw new RuntimeException("Grok statement produced a null message. Original message was: " - + originalMessage + " and the parsed message was: " + message + " . Check the pattern at: " - + grokPath); + + originalMessage + " , parsed message was: " + message + " , pattern was: " + + grokPattern); message.put("original_string", originalMessage); for (String timeField : timeFields) { @@ -199,6 +188,16 @@ public boolean validate(JSONObject message) { return false; } + protected boolean isGrokPatternUpdated(SensorParserConfig sensorParserConfig) { + Map parserConfig = sensorParserConfig.getParserConfig(); + return parserConfig != null && !Objects.equals(grokPattern, parserConfig.get("grokPattern")); + } + + protected boolean isPatternLabelUpdated(SensorParserConfig sensorParserConfig) { + Map parserConfig = sensorParserConfig.getParserConfig(); + return parserConfig != null && !Objects.equals(patternLabel, parserConfig.get("patternLabel")); + } + protected void postParse(JSONObject message) {} protected long toEpoch(String datetime) throws ParseException { diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java index 4f1c8b0081..1bc2fe080f 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java @@ -21,6 +21,7 @@ import oi.thekraken.grok.api.Match; import oi.thekraken.grok.api.exception.GrokException; import org.apache.commons.io.IOUtils; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; @@ -230,7 +231,7 @@ public void init() { } @Override - public List parse(byte[] raw_message) { + public List parse(byte[] raw_message, SensorParserConfig sensorParserConfig) { String toParse = ""; JSONObject toReturn; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 34dff11eed..a964aa98e5 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -117,7 +117,7 @@ public void execute(Tuple tuple) { int numWritten = 0; if(sensorParserConfig != null) { List fieldValidations = getConfigurations().getFieldValidations(); - Optional> messages = parser.parseOptional(originalMessage); + Optional> messages = parser.parseOptional(originalMessage, sensorParserConfig); for (JSONObject message : messages.orElse(Collections.emptyList())) { if (parser.validate(message) && filter != null && filter.emitTuple(message, stellarContext)) { message.put(Constants.SENSOR_TYPE, getSensorType()); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java index df522050ba..b8e7587ede 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java @@ -19,6 +19,7 @@ package org.apache.metron.parsers.bro; import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -54,7 +55,7 @@ public void init() { } @SuppressWarnings("unchecked") - public List parse(byte[] msg) { + public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { _LOG.trace("[Metron] Starting to parse incoming message"); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java index 19a768e310..ef1817c32a 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java @@ -19,6 +19,7 @@ package org.apache.metron.parsers.csv; import com.google.common.collect.ImmutableList; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.csv.CSVConverter; import org.apache.metron.common.utils.ConversionUtils; import org.apache.metron.parsers.BasicParser; @@ -53,7 +54,7 @@ public void init() { @Override - public List parse(byte[] rawMessage) { + public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { try { String msg = new String(rawMessage, "UTF-8"); Map value = converter.toMap(msg); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java index 498248a517..80041821e6 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java @@ -21,6 +21,7 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import org.apache.commons.lang3.StringUtils; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.utils.ParserUtils; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; @@ -70,7 +71,7 @@ public void init() { } @Override - public List parse(byte[] raw_message) { + public List parse(byte[] raw_message, SensorParserConfig sensorParserConfig) { String toParse = ""; List messages = new ArrayList<>(); try { diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java index e3b903ee07..2af701b5c4 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java @@ -17,9 +17,9 @@ */ package org.apache.metron.parsers.interfaces; -import java.io.Serializable; +import org.apache.metron.common.configuration.SensorParserConfig; + import java.util.List; -import java.util.Map; import java.util.Optional; public interface MessageParser extends Configurable { @@ -34,15 +34,15 @@ public interface MessageParser extends Configurable { * @param rawMessage * @return If null is returned, this is treated as an empty list. */ - List parse(byte[] rawMessage); + List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig); /** * Take raw data and convert it to an optional list of messages. * @param parseMessage * @return If null is returned, this is treated as an empty list. */ - default Optional> parseOptional(byte[] parseMessage) { - return Optional.ofNullable(parse(parseMessage)); + default Optional> parseOptional(byte[] parseMessage, SensorParserConfig sensorParserConfig) { + return Optional.ofNullable(parse(parseMessage, sensorParserConfig)); } /** diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java index 2d559ac6cf..7463414e8f 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java @@ -20,6 +20,7 @@ package org.apache.metron.parsers.ise; import com.esotericsoftware.minlog.Log; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -49,7 +50,7 @@ public void init() { @SuppressWarnings("unchecked") @Override - public List parse(byte[] msg) { + public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { String raw_message = ""; List messages = new ArrayList<>(); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java index 5d824561dc..c3103ae8c8 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; @@ -92,7 +93,7 @@ public void init() { * @return If null is returned, this is treated as an empty list. */ @Override - public List parse(byte[] rawMessage) { + public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { try { String originalString = new String(rawMessage); //convert the JSON blob into a String -> Object map diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java index 83eedcc7f9..36c8c3a5bb 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java @@ -18,6 +18,7 @@ package org.apache.metron.parsers.lancope; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.json.simple.JSONValue; @@ -50,7 +51,7 @@ public void init() { //@SuppressWarnings("unchecked") @Override - public List parse(byte[] msg) { + public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { JSONObject payload = null; List messages = new ArrayList<>(); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java index 2f5310c9d6..c220fde30f 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java @@ -17,6 +17,7 @@ */ package org.apache.metron.parsers.logstash; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -39,7 +40,7 @@ public void init() { } @Override - public List parse(byte[] raw_message) { + public List parse(byte[] raw_message, SensorParserConfig sensorParserConfig) { List messages = new ArrayList<>(); try { diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java index e6b927414d..d5615189bc 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java @@ -18,6 +18,7 @@ package org.apache.metron.parsers.paloalto; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -102,7 +103,7 @@ public void init() { } @SuppressWarnings({"unchecked", "unused"}) - public List parse(byte[] msg) { + public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { JSONObject outputMessage = new JSONObject(); String toParse = ""; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java index e50926cdc6..0e2a122c21 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java @@ -21,6 +21,7 @@ import jdk.nashorn.internal.runtime.arrays.ArrayIndex; import org.apache.metron.common.Constants; import org.apache.metron.common.csv.CSVConverter; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -98,7 +99,7 @@ public void init() { } @Override - public List parse(byte[] rawMessage) { + public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { JSONObject jsonMessage = new JSONObject(); List messages = new ArrayList<>(); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java index 0bc2671aa7..b57995d070 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java @@ -18,6 +18,7 @@ package org.apache.metron.parsers.sourcefire; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -53,7 +54,7 @@ public void init() { } @SuppressWarnings({ "unchecked", "unused" }) - public List parse(byte[] msg) { + public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { JSONObject payload = new JSONObject(); String toParse = ""; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java index 9590d34219..5d35ba4821 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java @@ -20,6 +20,7 @@ import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import junit.framework.Assert; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -37,11 +38,13 @@ public abstract class GrokParserTest { public void test() throws IOException, ParseException { Map parserConfig = new HashMap<>(); - parserConfig.put("grokPath", getGrokPath()); + parserConfig.put("grokPattern", getGrokPattern()); parserConfig.put("patternLabel", getGrokPatternLabel()); parserConfig.put("timestampField", getTimestampField()); parserConfig.put("dateFormat", getDateFormat()); parserConfig.put("timeFields", getTimeFields()); + SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setParserConfig(parserConfig); GrokParser grokParser = new GrokParser(); grokParser.configure(parserConfig); @@ -54,7 +57,7 @@ public void test() throws IOException, ParseException { JSONObject expected = (JSONObject) jsonParser.parse(e.getValue()); byte[] rawMessage = e.getKey().getBytes(); - List parsedList = grokParser.parse(rawMessage); + List parsedList = grokParser.parse(rawMessage, sensorParserConfig); Assert.assertEquals(1, parsedList.size()); compare(expected, parsedList.get(0)); } @@ -87,7 +90,7 @@ public boolean compare(JSONObject expected, JSONObject actual) { } public abstract Map getTestData(); - public abstract String getGrokPath(); + public abstract String getGrokPattern(); public abstract String getGrokPatternLabel(); public abstract List getTimeFields(); public abstract String getDateFormat(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java index 1d2af78615..0888dfe44c 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java @@ -19,6 +19,7 @@ package org.apache.metron.parsers; import junit.framework.Assert; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.interfaces.MessageParser; import org.junit.Test; @@ -37,7 +38,7 @@ public void init() { } @Override - public List parse(byte[] rawMessage) { + public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { return null; } @@ -51,8 +52,8 @@ public void configure(Map config) { } }; - Assert.assertNotNull(parser.parseOptional(null)); - Assert.assertFalse(parser.parseOptional(null).isPresent()); + Assert.assertNotNull(parser.parseOptional(null, null)); + Assert.assertFalse(parser.parseOptional(null, null).isPresent()); } @Test @@ -64,7 +65,7 @@ public void init() { } @Override - public List parse(byte[] rawMessage) { + public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { return new ArrayList<>(); } @@ -78,8 +79,8 @@ public void configure(Map config) { } }; - Assert.assertNotNull(parser.parseOptional(null)); - Optional ret = parser.parseOptional(null); + Assert.assertNotNull(parser.parseOptional(null, null)); + Optional ret = parser.parseOptional(null, null); Assert.assertTrue(ret.isPresent()); Assert.assertEquals(0, ret.get().size()); } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java index 89eb30c24f..5895276aff 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java @@ -17,7 +17,11 @@ */ package org.apache.metron.parsers; +import junit.framework.Assert; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.json.simple.JSONObject; +import org.junit.Test; import java.util.ArrayList; import java.util.HashMap; @@ -68,8 +72,9 @@ public Map getTestData() { } - public String getGrokPath() { - return "../metron-integration-test/src/main/sample/patterns/test"; + public String getGrokPattern() { + return "YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}\n" + + "YAF_DELIMITED %{NUMBER:start_time}\\|%{YAF_TIME_FORMAT:end_time}\\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\\|%{SPACE:UNWANTED}%{INT:protocol}\\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\\|%{SPACE:UNWANTED}%{INT:ip_src_port}\\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\\|%{SPACE:UNWANTED}%{DATA:iflags}\\|%{SPACE:UNWANTED}%{DATA:uflags}\\|%{SPACE:UNWANTED}%{DATA:riflags}\\|%{SPACE:UNWANTED}%{DATA:ruflags}\\|%{SPACE:UNWANTED}%{WORD:isn}\\|%{SPACE:UNWANTED}%{DATA:risn}\\|%{SPACE:UNWANTED}%{DATA:tag}\\|%{GREEDYDATA:rtag}\\|%{SPACE:UNWANTED}%{INT:pkt}\\|%{SPACE:UNWANTED}%{INT:oct}\\|%{SPACE:UNWANTED}%{INT:rpkt}\\|%{SPACE:UNWANTED}%{INT:roct}\\|%{SPACE:UNWANTED}%{INT:app}\\|%{GREEDYDATA:end_reason}"; } public String getGrokPatternLabel() { @@ -89,4 +94,36 @@ public String getDateFormat() { public String getTimestampField() { return "start_time"; } + + @Test + public void testConfigChange() { + String raw = "123 test"; + String pattern1 = "LABEL %{NUMBER:field_1}"; + String pattern2 = "LABEL %{NUMBER:field_1} %{WORD:field_2}"; + JSONObject expected1 = new JSONObject(); + expected1.put("field_1", 123); + expected1.put("original_string", raw); + JSONObject expected2 = new JSONObject(); + expected2.put("field_1", 123); + expected2.put("field_2", "test"); + expected2.put("original_string", raw); + Map parserConfig = new HashMap<>(); + parserConfig.put("grokPattern", pattern1); + parserConfig.put("patternLabel", "LABEL"); + SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setParserConfig(parserConfig); + + GrokParser grokParser = new GrokParser(); + grokParser.configure(parserConfig); + grokParser.init(); + + List results = grokParser.parse(raw.getBytes(), sensorParserConfig); + Assert.assertEquals(1, results.size()); + compare(expected1, results.get(0)); + + parserConfig.put("grokPattern", pattern2); + results = grokParser.parse(raw.getBytes(), sensorParserConfig); + Assert.assertEquals(1, results.size()); + compare(expected2, results.get(0)); + } } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java index 8114e836df..e621abb070 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java @@ -19,6 +19,7 @@ package org.apache.metron.parsers; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.snort.BasicSnortParser; import org.junit.Assert; import org.junit.Test; @@ -32,12 +33,14 @@ public class SnortParserTest { @Multiline public static String goodMessage; + private SensorParserConfig sensorParserConfig = new SensorParserConfig(); + @Test public void testGoodMessage() { BasicSnortParser parser = new BasicSnortParser(); parser.init(); - Map out = parser.parse(goodMessage.getBytes()).get(0); + Map out = parser.parse(goodMessage.getBytes(), sensorParserConfig).get(0); Assert.assertEquals(out.get("msg"),"Consecutive TCP small segments, exceeding threshold"); Assert.assertEquals(out.get("sig_rev"), "1"); Assert.assertEquals(out.get("ip_dst_addr"), "10.0.2.15"); @@ -71,6 +74,6 @@ public void testGoodMessage() { public void testBadMessage() { BasicSnortParser parser = new BasicSnortParser(); parser.init(); - parser.parse("foo bar".getBytes()); + parser.parse("foo bar".getBytes(), sensorParserConfig); } } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java index 93c8276a2b..846e55f402 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java @@ -75,8 +75,8 @@ public Map getTestData() { @Override - public String getGrokPath() { - return "../metron-parsers/src/main/resources/patterns/squid"; + public String getGrokPattern() { + return "SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?"; } @Override diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java index 8dd75a02d5..c6f1c9fa68 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java @@ -69,8 +69,9 @@ public Map getTestData() { } @Override - public String getGrokPath() { - return "../metron-parsers/src/main/resources/patterns/yaf"; + public String getGrokPattern() { + return "YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}\n" + + "YAF_DELIMITED %{YAF_TIME_FORMAT:start_time}\\|%{YAF_TIME_FORMAT:end_time}\\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\\|%{SPACE:UNWANTED}%{INT:protocol}\\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\\|%{SPACE:UNWANTED}%{INT:ip_src_port}\\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\\|%{SPACE:UNWANTED}%{DATA:iflags}\\|%{SPACE:UNWANTED}%{DATA:uflags}\\|%{SPACE:UNWANTED}%{DATA:riflags}\\|%{SPACE:UNWANTED}%{DATA:ruflags}\\|%{SPACE:UNWANTED}%{WORD:isn}\\|%{SPACE:UNWANTED}%{DATA:risn}\\|%{SPACE:UNWANTED}%{DATA:tag}\\|%{GREEDYDATA:rtag}\\|%{SPACE:UNWANTED}%{INT:pkt}\\|%{SPACE:UNWANTED}%{INT:oct}\\|%{SPACE:UNWANTED}%{INT:rpkt}\\|%{SPACE:UNWANTED}%{INT:roct}\\|%{SPACE:UNWANTED}%{INT:app}\\|%{GREEDYDATA:end_reason}"; } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java index 8e9da0d65e..fb3fdce952 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java @@ -20,6 +20,7 @@ import java.util.Iterator; import java.util.Map; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.sourcefire.BasicSourcefireParser; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -92,13 +93,13 @@ public void tearDown() throws Exception { } /** - * Test method for {@link BasicSourcefireParser#parse(byte[])}. + * Test method for {@link BasicSourcefireParser#parse(byte[], SensorParserConfig)}. */ @SuppressWarnings({ "rawtypes" }) public void testParse() { for (String grokAsaString : getGrokAsaStrings()) { - JSONObject parsed = grokAsaParser.parse(grokAsaString.getBytes()).get(0); + JSONObject parsed = grokAsaParser.parse(grokAsaString.getBytes(), new SensorParserConfig()).get(0); Assert.assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index 1f62e4f754..b39d977660 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -80,19 +80,20 @@ public class ParserBoltTest extends BaseBoltTest { @Test public void testEmpty() throws Exception { String sensorType = "yaf"; + SensorParserConfig sensorParserConfig = new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + }}; + } + }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - }}; - } - }; + return sensorParserConfig; } }; } @@ -106,7 +107,7 @@ public Map getParserConfig() { byte[] sampleBinary = "some binary message".getBytes(); when(tuple.getBinary(0)).thenReturn(sampleBinary); - when(parser.parseOptional(sampleBinary)).thenReturn(null); + when(parser.parseOptional(sampleBinary, sensorParserConfig)).thenReturn(null); parserBolt.execute(tuple); verify(parser, times(0)).validate(any()); verify(writer, times(0)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), any()); @@ -117,19 +118,20 @@ public Map getParserConfig() { public void test() throws Exception { String sensorType = "yaf"; + SensorParserConfig sensorParserConfig = new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + }}; + } + }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - }}; - } - }; + return sensorParserConfig; } }; } @@ -151,7 +153,7 @@ public Map getParserConfig() { final JSONObject finalMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"source.type\":\"" + sensorType + "\" }"); final JSONObject finalMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"source.type\":\"" + sensorType + "\" }"); when(tuple.getBinary(0)).thenReturn(sampleBinary); - when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messages)); + when(parser.parseOptional(sampleBinary, sensorParserConfig)).thenReturn(Optional.of(messages)); when(parser.validate(eq(messages.get(0)))).thenReturn(true); when(parser.validate(eq(messages.get(1)))).thenReturn(false); parserBolt.execute(tuple); @@ -169,63 +171,67 @@ public Map getParserConfig() { parserBolt.execute(tuple); verify(outputCollector, times(1)).reportError(any(Throwable.class)); } -@Test -public void testImplicitBatchOfOne() throws Exception { - - String sensorType = "yaf"; - - ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { - @Override - protected ParserConfigurations defaultConfigurations() { - return new ParserConfigurations() { - @Override - public SensorParserConfig getSensorParserConfig(String sensorType) { - return new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - }}; - } - }; - } - }; - } - }; - parserBolt.setCuratorFramework(client); - parserBolt.setTreeCache(cache); - parserBolt.prepare(new HashMap(), topologyContext, outputCollector); - verify(parser, times(1)).init(); - verify(batchWriter, times(1)).init(any(), any()); - when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); - when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); - parserBolt.withMessageFilter(filter); - parserBolt.execute(t1); - verify(outputCollector, times(1)).ack(t1); -} - /** - { - "filterClassName" : "QUERY" - ,"parserConfig" : { - "filter.query" : "exists(field1)" - } - } - */ - @Multiline - public static String sensorParserConfig; + @Test + public void testImplicitBatchOfOne() throws Exception { + + String sensorType = "yaf"; + SensorParserConfig sensorParserConfig = new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + }}; + } + }; + ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { + @Override + protected ParserConfigurations defaultConfigurations() { + return new ParserConfigurations() { + @Override + public SensorParserConfig getSensorParserConfig(String sensorType) { + return sensorParserConfig; + } + }; + } + }; + parserBolt.setCuratorFramework(client); + parserBolt.setTreeCache(cache); + parserBolt.prepare(new HashMap(), topologyContext, outputCollector); + verify(parser, times(1)).init(); + verify(batchWriter, times(1)).init(any(), any()); + when(parser.validate(any())).thenReturn(true); + when(parser.parseOptional(any(), eq(sensorParserConfig))).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); + parserBolt.withMessageFilter(filter); + parserBolt.execute(t1); + verify(outputCollector, times(1)).ack(t1); + } + @Test public void testFilter() throws Exception { String sensorType = "yaf"; + SensorParserConfig sensorParserConfig = new SensorParserConfig() { + @Override + public String getFilterClassName() { + return "QUERY"; + } + @Override + public Map getParserConfig() { + return new HashMap() {{ + put("filter.query", "exists(field1)"); + }}; + } + }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override - protected SensorParserConfig getSensorParserConfig() { - try { - return SensorParserConfig.fromBytes(Bytes.toBytes(sensorParserConfig)); - } catch (IOException e) { - throw new RuntimeException(e); - } + protected ParserConfigurations defaultConfigurations() { + return new ParserConfigurations() { + @Override + public SensorParserConfig getSensorParserConfig(String sensorType) { + return sensorParserConfig; + } + }; } }; parserBolt.setCuratorFramework(client); @@ -234,7 +240,7 @@ protected SensorParserConfig getSensorParserConfig() { verify(parser, times(1)).init(); verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(parser.parseOptional(any(), eq(sensorParserConfig))).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); verify(outputCollector, times(1)).ack(t1); @@ -243,21 +249,21 @@ protected SensorParserConfig getSensorParserConfig() { public void testBatchOfOne() throws Exception { String sensorType = "yaf"; - + SensorParserConfig sensorParserConfig = new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + put(ParserWriterConfiguration.BATCH_CONF, "1"); + }}; + } + }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - put(ParserWriterConfiguration.BATCH_CONF, "1"); - }}; - } - }; + return sensorParserConfig; } }; } @@ -268,7 +274,7 @@ public Map getParserConfig() { verify(parser, times(1)).init(); verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(parser.parseOptional(any(), eq(sensorParserConfig))).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); @@ -278,21 +284,21 @@ public Map getParserConfig() { public void testBatchOfFive() throws Exception { String sensorType = "yaf"; - + SensorParserConfig sensorParserConfig = new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + put(ParserWriterConfiguration.BATCH_CONF, 5); + }}; + } + }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - put(ParserWriterConfiguration.BATCH_CONF, 5); - }}; - } - }; + return sensorParserConfig; } }; } @@ -303,7 +309,7 @@ public Map getParserConfig() { verify(parser, times(1)).init(); verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(parser.parseOptional(any(), eq(sensorParserConfig))).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); writeNonBatch(outputCollector, parserBolt, t1); @@ -323,20 +329,21 @@ public Map getParserConfig() { public void testBatchOfFiveWithError() throws Exception { String sensorType = "yaf"; + SensorParserConfig sensorParserConfig = new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + put(ParserWriterConfiguration.BATCH_CONF, 5); + }}; + } + }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - put(ParserWriterConfiguration.BATCH_CONF, 5); - }}; - } - }; + return sensorParserConfig; } }; } @@ -349,7 +356,7 @@ public Map getParserConfig() { doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject())); + when(parser.parse(any(), eq(sensorParserConfig))).thenReturn(ImmutableList.of(new JSONObject())); when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java index 55c6695af0..3a5aa8efbb 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java @@ -19,6 +19,7 @@ import junit.framework.TestCase; import org.apache.commons.lang3.tuple.Pair; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -34,6 +35,7 @@ public class BasicBroParserTest extends TestCase { */ private BasicBroParser broParser = null; private JSONParser jsonParser = null; + private SensorParserConfig sensorParserConfig = null; /** * Constructs a new BasicBroParserTest instance. @@ -43,6 +45,7 @@ public class BasicBroParserTest extends TestCase { public BasicBroParserTest() throws Exception { broParser = new BasicBroParser(); jsonParser = new JSONParser(); + sensorParserConfig = new SensorParserConfig(); } /** @@ -70,7 +73,7 @@ public void testUnwrappedBroMessage() throws ParseException { JSONObject rawJson = (JSONObject)jsonParser.parse(rawMessage); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedBroTimestamp = "1449511228.474"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); @@ -96,7 +99,7 @@ public void testHttpBroMessage() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedBroTimestamp = "1402307733.473"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1402307733473"; @@ -127,7 +130,7 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedTimestamp = "1467657279000"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); String expectedBroTimestamp = "1467657279.0"; @@ -140,7 +143,7 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedTimestamp = "1467657279000"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); String expectedBroTimestamp = "1467657279.0"; @@ -153,7 +156,7 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedTimestamp = "1467657279100"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); String expectedBroTimestamp = "1467657279.1"; @@ -166,7 +169,7 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedTimestamp = "1467657279110"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); String expectedBroTimestamp = "1467657279.11"; @@ -180,7 +183,7 @@ public void testHttpDecimalBroMessage() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedBroTimestamp = "1457149494.166991"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1457149494166"; @@ -205,7 +208,7 @@ public void testDnsBroMessage() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedBroTimestamp = "1402308259.609"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1402308259609"; @@ -227,7 +230,7 @@ public void testFilesBroMessage() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedBroTimestamp = "1425845251.334"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1425845251334"; @@ -248,7 +251,7 @@ public void testProtocolKeyCleanedUp() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); String expectedBroTimestamp = "1402307733.473"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1402307733473"; @@ -259,7 +262,7 @@ public void testProtocolKeyCleanedUp() throws ParseException { public void testBadMessage() throws ParseException{ try { - broParser.parse("{ \"foo\" : \"bar\"}".getBytes()); + broParser.parse("{ \"foo\" : \"bar\"}".getBytes(), sensorParserConfig); Assert.fail("Should have marked this as a bad message."); } catch(IllegalStateException ise) { @@ -267,7 +270,7 @@ public void testBadMessage() throws ParseException{ } //non json try { - broParser.parse("foo bar".getBytes()); + broParser.parse("foo bar".getBytes(), sensorParserConfig); Assert.fail("Should have marked this as a bad message."); } catch(IllegalStateException ise) { diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java index 2dd11c59bc..b7a3b99280 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java @@ -24,6 +24,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -61,14 +62,14 @@ public class BroParserTest extends AbstractConfigTest { * The parser. */ private BasicBroParser parser=null; - + /** * Constructs a new BroParserTest instance. - * @throws Exception + * @throws Exception */ public BroParserTest() throws Exception { super(); - } + } /** @@ -101,7 +102,7 @@ public void setUp() throws Exception { public void testParse() throws ParseException { for (String inputString : getInputStrings()) { - JSONObject cleanJson = parser.parse(inputString.getBytes()).get(0); + JSONObject cleanJson = parser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); Assert.assertNotNull(cleanJson); System.out.println(cleanJson); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java index e667e54645..cfa5d4d1d4 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java @@ -56,15 +56,15 @@ public void test() throws IOException { parser.configure(config.getParserConfig()); { String line = "#foo,bar,grok"; - Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size()); + Assert.assertEquals(0, parser.parse(Bytes.toBytes(line), config).size()); } { String line = ""; - Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size()); + Assert.assertEquals(0, parser.parse(Bytes.toBytes(line), config).size()); } { String line = "foo,bar,grok"; - List results = parser.parse(Bytes.toBytes(line)); + List results = parser.parse(Bytes.toBytes(line), config); Assert.assertEquals(1, results.size()); JSONObject o = results.get(0); Assert.assertTrue(parser.validate(o)); @@ -75,7 +75,7 @@ public void test() throws IOException { } { String line = "\"foo\", \"bar\",\"grok\""; - List results = parser.parse(Bytes.toBytes(line)); + List results = parser.parse(Bytes.toBytes(line), config); Assert.assertEquals(1, results.size()); JSONObject o = results.get(0); Assert.assertTrue(parser.validate(o)); @@ -86,7 +86,7 @@ public void test() throws IOException { } { String line = "foo, bar, grok"; - List results = parser.parse(Bytes.toBytes(line)); + List results = parser.parse(Bytes.toBytes(line), config); Assert.assertEquals(1, results.size()); JSONObject o = results.get(0); Assert.assertTrue(parser.validate(o)); @@ -98,7 +98,7 @@ public void test() throws IOException { { String line = "foo"; try { - List results = parser.parse(Bytes.toBytes(line)); + List results = parser.parse(Bytes.toBytes(line), config); Assert.fail("Expected exception"); } catch(IllegalStateException iae) {} diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java index 129619d6c9..006693ed1e 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.Map; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -97,12 +98,12 @@ public void tearDown() throws Exception { * * * - * {@link BasicFireEyeParser#parse(byte[])}. + * {@link BasicFireEyeParser#parse(byte[], SensorParserConfig)}. */ @SuppressWarnings({ "rawtypes"}) public void testParse() { for (String inputString : getInputStrings()) { - JSONObject parsed = parser.parse(inputString.getBytes()).get(0); + JSONObject parsed = parser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); Assert.assertNotNull(parsed); JSONParser parser = new JSONParser(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java index 751e4147f4..cc084d59d5 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java @@ -21,6 +21,7 @@ import java.net.URL; import java.util.Map; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -35,7 +36,7 @@ *
  • Description: Junit Test Case for BasicISE Parser
    • *
  • Created: AUG 25, 2014
    • * - * + * * @version $Revision: 1.1 $ */ @@ -43,7 +44,7 @@ public class BasicIseParserTest extends AbstractSchemaTest { /** * The inputStrings. */ - private static String[] inputStrings; + private static String[] inputStrings; /** * The parser. @@ -53,7 +54,7 @@ public class BasicIseParserTest extends AbstractSchemaTest { /** * Constructs a new BasicIseParserTest instance. - * + * * @param name */ @@ -62,14 +63,14 @@ public BasicIseParserTest(String name) { } /** - * + * * @throws java.lang.Exception */ protected static void setUpBeforeClass() throws Exception { } /** - * + * * @throws java.lang.Exception */ protected static void tearDownAfterClass() throws Exception { @@ -77,7 +78,7 @@ protected static void tearDownAfterClass() throws Exception { /* * (non-Javadoc) - * + * * @see junit.framework.TestCase#setUp() */ @@ -85,7 +86,7 @@ protected void setUp() throws Exception { super.setUp("org.apache.metron.parsers.lancope.BasicLancopeParserTest"); setInputStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); BasicIseParserTest.setIseParser(new BasicIseParser()); - + URL schema_url = getClass().getClassLoader().getResource( "TestSchemas/IseSchema.json"); super.setSchemaJsonString(super.readSchemaFromFile(schema_url)); @@ -93,7 +94,7 @@ protected void setUp() throws Exception { /* * (non-Javadoc) - * + * * @see junit.framework.TestCase#tearDown() */ @@ -103,14 +104,14 @@ protected void tearDown() throws Exception { /** * Test method for - * {@link BasicIseParser#parse(byte[])}. - * + * {@link BasicIseParser#parse(byte[], SensorParserConfig)}. + * * @throws IOException * @throws Exception */ public void testParse() throws ParseException, IOException, Exception { for (String inputString : getInputStrings()) { - JSONObject parsed = parser.parse(inputString.getBytes()).get(0); + JSONObject parsed = parser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java index 61748e7e14..6006f6511d 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; @@ -41,7 +42,7 @@ public class JSONMapParserTest { @Test public void testHappyPath() { JSONMapParser parser = new JSONMapParser(); - List output = parser.parse(happyPathJSON.getBytes()); + List output = parser.parse(happyPathJSON.getBytes(), sensorParserConfig); Assert.assertEquals(output.size(), 1); //don't forget the timestamp field! Assert.assertEquals(output.get(0).size(), 5); @@ -73,10 +74,12 @@ public void testHappyPath() { @Multiline static String mixCollectionHandlingJSON; + private SensorParserConfig sensorParserConfig = new SensorParserConfig(); + @Test public void testCollectionHandlingDrop() { JSONMapParser parser = new JSONMapParser(); - List output = parser.parse(collectionHandlingJSON.getBytes()); + List output = parser.parse(collectionHandlingJSON.getBytes(), sensorParserConfig); Assert.assertEquals(output.size(), 1); //don't forget the timestamp field! Assert.assertEquals(output.get(0).size(), 2); @@ -89,7 +92,7 @@ public void testCollectionHandlingDrop() { public void testCollectionHandlingError() { JSONMapParser parser = new JSONMapParser(); parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ERROR.name())); - parser.parse(collectionHandlingJSON.getBytes()); + parser.parse(collectionHandlingJSON.getBytes(), sensorParserConfig); } @@ -97,7 +100,7 @@ public void testCollectionHandlingError() { public void testCollectionHandlingAllow() { JSONMapParser parser = new JSONMapParser(); parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ALLOW.name())); - List output = parser.parse(collectionHandlingJSON.getBytes()); + List output = parser.parse(collectionHandlingJSON.getBytes(), sensorParserConfig); Assert.assertEquals(output.size(), 1); //don't forget the timestamp field! Assert.assertEquals(output.get(0).size(), 3); @@ -110,7 +113,7 @@ public void testCollectionHandlingAllow() { public void testCollectionHandlingUnfold() { JSONMapParser parser = new JSONMapParser(); parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.UNFOLD.name())); - List output = parser.parse(collectionHandlingJSON.getBytes()); + List output = parser.parse(collectionHandlingJSON.getBytes(), sensorParserConfig); Assert.assertEquals(output.size(), 1); //don't forget the timestamp field! Assert.assertEquals(output.get(0).size(), 6); @@ -127,7 +130,7 @@ public void testCollectionHandlingUnfold() { public void testMixedCollectionHandlingUnfold() { JSONMapParser parser = new JSONMapParser(); parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG,JSONMapParser.MapStrategy.UNFOLD.name())); - List output = parser.parse(mixCollectionHandlingJSON.getBytes()); + List output = parser.parse(mixCollectionHandlingJSON.getBytes(), sensorParserConfig); Assert.assertEquals(output.get(0).size(), 4); JSONObject message = output.get(0); Assert.assertEquals(message.get("collection.key"), "value"); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java index 4d8a2d0ff2..ba4638676e 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java @@ -21,6 +21,7 @@ import java.net.URL; import java.util.Map; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -37,17 +38,17 @@ * @version $Revision: 1.1 $ */ public class BasicLancopeParserTest extends AbstractSchemaTest { - + /** * The inputStrings. */ - private static String[] inputStrings; + private static String[] inputStrings; /** * The parser. */ - private static BasicLancopeParser parser=null; + private static BasicLancopeParser parser=null; /** * Constructs a new BasicLancopeParserTest instance. @@ -59,20 +60,20 @@ public BasicLancopeParserTest(String name) { } /** - + * @throws java.lang.Exception */ - protected static void setUpBeforeClass() throws Exception { + protected static void setUpBeforeClass() throws Exception { } /** - + * @throws java.lang.Exception */ protected static void tearDownAfterClass() throws Exception { } - /* + /* * (non-Javadoc) * @see junit.framework.TestCase#setUp() */ @@ -80,14 +81,14 @@ protected static void tearDownAfterClass() throws Exception { protected void setUp() throws Exception { super.setUp("org.apache.metron.parsers.lancope.BasicLancopeParserTest"); setInputStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - BasicLancopeParserTest.setParser(new BasicLancopeParser()); - + BasicLancopeParserTest.setParser(new BasicLancopeParser()); + URL schema_url = getClass().getClassLoader().getResource( "TestSchemas/LancopeSchema.json"); - super.setSchemaJsonString(super.readSchemaFromFile(schema_url)); + super.setSchemaJsonString(super.readSchemaFromFile(schema_url)); } - /* + /* * (non-Javadoc) * @see junit.framework.TestCase#tearDown() */ @@ -97,14 +98,14 @@ protected void tearDown() throws Exception { } /** - * Test method for {@link BasicLancopeParser#parse(byte[])}. - * @throws Exception - * @throws IOException + * Test method for {@link BasicLancopeParser#parse(byte[], SensorParserConfig)}. + * @throws Exception + * @throws IOException */ public void testParse() throws IOException, Exception { - + for (String inputString : getInputStrings()) { - JSONObject parsed = parser.parse(inputString.getBytes()).get(0); + JSONObject parsed = parser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java index 0c0947b2a6..ac9c89ca89 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java @@ -20,6 +20,7 @@ import java.util.Iterator; import java.util.Map; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.sourcefire.BasicSourcefireParser; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -90,12 +91,12 @@ public void tearDown() throws Exception { /** * Test method for - * {@link BasicSourcefireParser#parse(byte[])}. + * {@link BasicSourcefireParser#parse(byte[], SensorParserConfig)}. */ @SuppressWarnings({ "rawtypes" }) public void testParse() { for (String inputString : getInputStrings()) { - JSONObject parsed = paParser.parse(inputString.getBytes()).get(0); + JSONObject parsed = paParser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); Assert.assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java index 2ce035c238..f032c6d27f 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.Map; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -91,13 +92,13 @@ public void tearDown() throws Exception { } /** - * Test method for {@link BasicSourcefireParser#parse(byte[])}. + * Test method for {@link BasicSourcefireParser#parse(byte[], SensorParserConfig)}. */ @SuppressWarnings({ "rawtypes", "unused" }) public void testParse() { for (String sourceFireString : getSourceFireStrings()) { byte[] srcBytes = sourceFireString.getBytes(); - JSONObject parsed = sourceFireParser.parse(sourceFireString.getBytes()).get(0); + JSONObject parsed = sourceFireParser.parse(sourceFireString.getBytes(), new SensorParserConfig()).get(0); Assert.assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java index 87afe10002..f215729541 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.junit.Before; import org.junit.Test; @@ -31,14 +32,53 @@ public class GrokWebSphereParserTest { private Map parserConfig; + private SensorParserConfig sensorParserConfig; @Before public void setup() { parserConfig = new HashMap<>(); - parserConfig.put("grokPath", "../metron-parsers/src/main/resources/patterns/websphere"); + parserConfig.put("grokPattern", "# Months - only three-letter code is used\n" + + "MONTH \\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec?)\\b\n" + + "\n" + + "# Days - two digit number is used\n" + + "DAY \\d{1,2}\n" + + "\n" + + "# Time - two digit hour, minute, and second\n" + + "TIME \\d{2}:\\d{2}:\\d{2}\n" + + "\n" + + "# Timestamp - month, day, and time\n" + + "TIMESTAMP %{MONTH:UNWANTED}\\s+%{DAY:UNWANTED} %{TIME:UNWANTED}\n" + + "\n" + + "# Generic word field\n" + + "WORD \\w+\n" + + "\n" + + "# Priority\n" + + "PRIORITY \\d+\n" + + "\n" + + "# Log start - the first part of the log line\n" + + "LOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname}\n" + + "\n" + + "# Security domain\n" + + "SECURITY_DOMAIN [%{WORD:security_domain}]\n" + + "\n" + + "# Log middle - the middle part of the log line\n" + + "LOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]\n" + + "\n" + + "# Define IP address formats\n" + + "IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?\n" + + "IPV4 (?Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] user(rick007): " + "[120.43.200.6]: User logged into 'cohlOut'."; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); JSONObject parsedJSON = result.get(0); //Compare fields @@ -73,7 +113,7 @@ public void tetsParseLogoutLine() throws Exception { parser.configure(parserConfig); String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201]: " + "User 'hjpotter' logged out from 'default'."; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); JSONObject parsedJSON = result.get(0); //Compare fields @@ -96,7 +136,7 @@ public void tetsParseRBMLine() throws Exception { parser.configure(parserConfig); String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbm(RBM-Settings): " + "trans(3502888135)[request] gtid(3502888135): RBM: Resource access denied."; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); JSONObject parsedJSON = result.get(0); //Compare fields @@ -118,7 +158,7 @@ public void tetsParseOtherLine() throws Exception { parser.configure(parserConfig); String testString = "<134>Apr 15 17:17:34 SAGPXMLQA333 [0x8240001c][audit][info] trans(191): (admin:default:system:*): " + "ntp-service 'NTP Service' - Operational state down"; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); JSONObject parsedJSON = result.get(0); //Compare fields @@ -140,7 +180,7 @@ public void testParseMalformedLoginLine() throws Exception { parser.configure(parserConfig); String testString = "<133>Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] rick007): " + "[120.43.200. User logged into 'cohlOut'."; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); JSONObject parsedJSON = result.get(0); //Compare fields @@ -164,7 +204,7 @@ public void tetsParseMalformedLogoutLine() throws Exception { parser.configure(parserConfig); String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201: " + "User 'hjpotter' logged out from 'default."; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); JSONObject parsedJSON = result.get(0); //Compare fields @@ -187,7 +227,7 @@ public void tetsParseMalformedRBMLine() throws Exception { parser.configure(parserConfig); String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbmRBM-Settings): " + "trans3502888135)[request] gtid3502888135) RBM: Resource access denied."; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); JSONObject parsedJSON = result.get(0); //Compare fields @@ -209,7 +249,7 @@ public void tetsParseMalformedOtherLine() throws Exception { parser.configure(parserConfig); String testString = "<134>Apr 15 17:17:34 SAGPXMLQA333 [0x8240001c][audit][info] trans 191) admindefaultsystem*): " + "ntp-service 'NTP Service' - Operational state down:"; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); JSONObject parsedJSON = result.get(0); //Compare fields @@ -232,7 +272,7 @@ public void testParseEmptyLine() throws Exception { GrokWebSphereParser parser = new GrokWebSphereParser(); parser.configure(parserConfig); String testString = ""; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes(), sensorParserConfig); } } From e2ac1f08089594f86dac4987876eddce96e8c880 Mon Sep 17 00:00:00 2001 From: rmerriman Date: Wed, 12 Oct 2016 17:19:38 -0500 Subject: [PATCH 2/7] Made a small correction to the wording in a log statement and removed unnecessary pattern files --- .../org/apache/metron/parsers/GrokParser.java | 2 +- .../src/main/resources/patterns/squid | 2 - .../src/main/resources/patterns/websphere | 37 ------------------- .../src/main/resources/patterns/yaf | 2 - 4 files changed, 1 insertion(+), 42 deletions(-) delete mode 100644 metron-platform/metron-parsers/src/main/resources/patterns/squid delete mode 100644 metron-platform/metron-parsers/src/main/resources/patterns/websphere delete mode 100644 metron-platform/metron-parsers/src/main/resources/patterns/yaf diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java index cd0b867776..80dba3607d 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java @@ -87,7 +87,7 @@ public void init() { if (commonInputStream == null) { throw new RuntimeException( - "Unable to initialize grok parser: Unable to load " + patternsCommonPath + " from either classpath or HDFS"); + "Unable to initialize grok parser: Unable to load " + patternsCommonPath + " from classpath"); } grok.addPatternFromReader(new InputStreamReader(commonInputStream)); diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/squid b/metron-platform/metron-parsers/src/main/resources/patterns/squid deleted file mode 100644 index bf6c5b7ee9..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/squid +++ /dev/null @@ -1,2 +0,0 @@ -SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})? - diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/websphere b/metron-platform/metron-parsers/src/main/resources/patterns/websphere deleted file mode 100644 index 546944c1c5..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/websphere +++ /dev/null @@ -1,37 +0,0 @@ -# Months - only three-letter code is used -MONTH \b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec?)\b - -# Days - two digit number is used -DAY \d{1,2} - -# Time - two digit hour, minute, and second -TIME \d{2}:\d{2}:\d{2} - -# Timestamp - month, day, and time -TIMESTAMP %{MONTH:UNWANTED}\s+%{DAY:UNWANTED} %{TIME:UNWANTED} - -# Generic word field -WORD \w+ - -# Priority -PRIORITY \d+ - -# Log start - the first part of the log line -LOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname} - -# Security domain -SECURITY_DOMAIN [%{WORD:security_domain}] - -# Log middle - the middle part of the log line -LOGMIDDLE (\[%{WORD:security_domain}\])?\[%{WORD:event_code}\]\[%{WORD:event_type}\]\[%{WORD:severity}\] - -# Define IP address formats -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (? Date: Thu, 13 Oct 2016 09:19:17 -0500 Subject: [PATCH 3/7] Improved GrokParser to support either a String or List of Strings in the grokPattern property --- .../config/zookeeper/parsers/websphere.json | 40 +++++++++--- .../main/config/zookeeper/parsers/yaf.json | 3 +- .../org/apache/metron/parsers/GrokParser.java | 12 +++- .../apache/metron/parsers/GrokParserTest.java | 2 +- .../metron/parsers/SampleGrokParserTest.java | 7 +- .../apache/metron/parsers/YafParserTest.java | 7 +- .../websphere/GrokWebSphereParserTest.java | 64 ++++++++----------- 7 files changed, 81 insertions(+), 54 deletions(-) diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json index 2a84f4403b..9d9b111934 100644 --- a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json +++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json @@ -1,11 +1,35 @@ { - "parserClassName":"org.apache.metron.parsers.websphere.GrokWebSphereParser", - "sensorTopic":"websphere", - "parserConfig": - { - "grokPattern":"# Days - two digit number is used\nDAY \\d{1,2}\n# Time - two digit hour, minute, and second\nTIME \\d{2}:\\d{2}:\\d{2}\n# Timestamp - month, day, and time\nTIMESTAMP %{MONTH:UNWANTED}\\s+%{DAY:UNWANTED} %{TIME:UNWANTED}\n# Generic word field\nWORD \\w+\n# Priority\nPRIORITY \\d+\n# Log start - the first part of the log line\nLOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname}\n# Security domain\nSECURITY_DOMAIN [%{WORD:security_domain}]\n# Log middle - the middle part of the log line\nLOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]\n# Define IP address formats\nIPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?\nIPV4 (??%{TIMESTAMP:timestamp_string} %{WORD:hostname}", + "# Security domain", + "SECURITY_DOMAIN [%{WORD:security_domain}]", + "# Log middle - the middle part of the log line", + "LOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]", + "# Define IP address formats", + "IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?", + "IPV4 (?, Serializable { protected SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z"); protected String patternsCommonPath = "/patterns/common"; + @SuppressWarnings("unchecked") @Override public void configure(Map parserConfig) { - this.grokPattern = (String) parserConfig.get("grokPattern"); + Object grokPattern = parserConfig.get("grokPattern"); + if (grokPattern instanceof String) { + this.grokPattern = (String) grokPattern; + } else if (grokPattern instanceof String[]){ + String[] patterns = (String[]) grokPattern; + this.grokPattern = Joiner.on('\n').join(patterns); + } else if (grokPattern instanceof Iterable) { + Iterable patterns = (Iterable) grokPattern; + this.grokPattern = Joiner.on('\n').join(patterns); + } this.patternLabel = (String) parserConfig.get("patternLabel"); this.timestampField = (String) parserConfig.get("timestampField"); List timeFieldsParam = (List) parserConfig.get("timeFields"); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java index 5d35ba4821..1275fc7a4a 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java @@ -90,7 +90,7 @@ public boolean compare(JSONObject expected, JSONObject actual) { } public abstract Map getTestData(); - public abstract String getGrokPattern(); + public abstract Object getGrokPattern(); public abstract String getGrokPatternLabel(); public abstract List getTimeFields(); public abstract String getDateFormat(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java index 5895276aff..6266985a5b 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java @@ -72,9 +72,10 @@ public Map getTestData() { } - public String getGrokPattern() { - return "YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}\n" + - "YAF_DELIMITED %{NUMBER:start_time}\\|%{YAF_TIME_FORMAT:end_time}\\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\\|%{SPACE:UNWANTED}%{INT:protocol}\\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\\|%{SPACE:UNWANTED}%{INT:ip_src_port}\\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\\|%{SPACE:UNWANTED}%{DATA:iflags}\\|%{SPACE:UNWANTED}%{DATA:uflags}\\|%{SPACE:UNWANTED}%{DATA:riflags}\\|%{SPACE:UNWANTED}%{DATA:ruflags}\\|%{SPACE:UNWANTED}%{WORD:isn}\\|%{SPACE:UNWANTED}%{DATA:risn}\\|%{SPACE:UNWANTED}%{DATA:tag}\\|%{GREEDYDATA:rtag}\\|%{SPACE:UNWANTED}%{INT:pkt}\\|%{SPACE:UNWANTED}%{INT:oct}\\|%{SPACE:UNWANTED}%{INT:rpkt}\\|%{SPACE:UNWANTED}%{INT:roct}\\|%{SPACE:UNWANTED}%{INT:app}\\|%{GREEDYDATA:end_reason}"; + public String[] getGrokPattern() { + String[] grokPattern = {"YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}", + "YAF_DELIMITED %{NUMBER:start_time}\\|%{YAF_TIME_FORMAT:end_time}\\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\\|%{SPACE:UNWANTED}%{INT:protocol}\\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\\|%{SPACE:UNWANTED}%{INT:ip_src_port}\\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\\|%{SPACE:UNWANTED}%{DATA:iflags}\\|%{SPACE:UNWANTED}%{DATA:uflags}\\|%{SPACE:UNWANTED}%{DATA:riflags}\\|%{SPACE:UNWANTED}%{DATA:ruflags}\\|%{SPACE:UNWANTED}%{WORD:isn}\\|%{SPACE:UNWANTED}%{DATA:risn}\\|%{SPACE:UNWANTED}%{DATA:tag}\\|%{GREEDYDATA:rtag}\\|%{SPACE:UNWANTED}%{INT:pkt}\\|%{SPACE:UNWANTED}%{INT:oct}\\|%{SPACE:UNWANTED}%{INT:rpkt}\\|%{SPACE:UNWANTED}%{INT:roct}\\|%{SPACE:UNWANTED}%{INT:app}\\|%{GREEDYDATA:end_reason}"}; + return grokPattern; } public String getGrokPatternLabel() { diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java index c6f1c9fa68..4c9af2ae34 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java @@ -69,9 +69,10 @@ public Map getTestData() { } @Override - public String getGrokPattern() { - return "YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}\n" + - "YAF_DELIMITED %{YAF_TIME_FORMAT:start_time}\\|%{YAF_TIME_FORMAT:end_time}\\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\\|%{SPACE:UNWANTED}%{INT:protocol}\\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\\|%{SPACE:UNWANTED}%{INT:ip_src_port}\\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\\|%{SPACE:UNWANTED}%{DATA:iflags}\\|%{SPACE:UNWANTED}%{DATA:uflags}\\|%{SPACE:UNWANTED}%{DATA:riflags}\\|%{SPACE:UNWANTED}%{DATA:ruflags}\\|%{SPACE:UNWANTED}%{WORD:isn}\\|%{SPACE:UNWANTED}%{DATA:risn}\\|%{SPACE:UNWANTED}%{DATA:tag}\\|%{GREEDYDATA:rtag}\\|%{SPACE:UNWANTED}%{INT:pkt}\\|%{SPACE:UNWANTED}%{INT:oct}\\|%{SPACE:UNWANTED}%{INT:rpkt}\\|%{SPACE:UNWANTED}%{INT:roct}\\|%{SPACE:UNWANTED}%{INT:app}\\|%{GREEDYDATA:end_reason}"; + public String[] getGrokPattern() { + String[] grokPattern = {"YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}", + "YAF_DELIMITED %{YAF_TIME_FORMAT:start_time}\\|%{YAF_TIME_FORMAT:end_time}\\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\\|%{SPACE:UNWANTED}%{INT:protocol}\\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\\|%{SPACE:UNWANTED}%{INT:ip_src_port}\\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\\|%{SPACE:UNWANTED}%{DATA:iflags}\\|%{SPACE:UNWANTED}%{DATA:uflags}\\|%{SPACE:UNWANTED}%{DATA:riflags}\\|%{SPACE:UNWANTED}%{DATA:ruflags}\\|%{SPACE:UNWANTED}%{WORD:isn}\\|%{SPACE:UNWANTED}%{DATA:risn}\\|%{SPACE:UNWANTED}%{DATA:tag}\\|%{GREEDYDATA:rtag}\\|%{SPACE:UNWANTED}%{INT:pkt}\\|%{SPACE:UNWANTED}%{INT:oct}\\|%{SPACE:UNWANTED}%{INT:rpkt}\\|%{SPACE:UNWANTED}%{INT:roct}\\|%{SPACE:UNWANTED}%{INT:app}\\|%{GREEDYDATA:end_reason}"}; + return grokPattern; } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java index f215729541..c82cafcb88 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java @@ -36,44 +36,34 @@ public class GrokWebSphereParserTest { @Before public void setup() { + String[] grokPattern = {"# Months - only three-letter code is used", + "MONTH \\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec?)\\b", + "# Days - two digit number is used", + "DAY \\d{1,2}", + "# Time - two digit hour, minute, and second", + "TIME \\d{2}:\\d{2}:\\d{2}", + "# Timestamp - month, day, and time", + "TIMESTAMP %{MONTH:UNWANTED}\\s+%{DAY:UNWANTED} %{TIME:UNWANTED}", + "# Generic word field", + "WORD \\w+", + "# Priority", + "PRIORITY \\d+", + "# Log start - the first part of the log line", + "LOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname}", + "# Security domain", + "SECURITY_DOMAIN [%{WORD:security_domain}]", + "# Log middle - the middle part of the log line", + "LOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]", + "# Define IP address formats", + "IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?", + "IPV4 (?(); - parserConfig.put("grokPattern", "# Months - only three-letter code is used\n" + - "MONTH \\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec?)\\b\n" + - "\n" + - "# Days - two digit number is used\n" + - "DAY \\d{1,2}\n" + - "\n" + - "# Time - two digit hour, minute, and second\n" + - "TIME \\d{2}:\\d{2}:\\d{2}\n" + - "\n" + - "# Timestamp - month, day, and time\n" + - "TIMESTAMP %{MONTH:UNWANTED}\\s+%{DAY:UNWANTED} %{TIME:UNWANTED}\n" + - "\n" + - "# Generic word field\n" + - "WORD \\w+\n" + - "\n" + - "# Priority\n" + - "PRIORITY \\d+\n" + - "\n" + - "# Log start - the first part of the log line\n" + - "LOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname}\n" + - "\n" + - "# Security domain\n" + - "SECURITY_DOMAIN [%{WORD:security_domain}]\n" + - "\n" + - "# Log middle - the middle part of the log line\n" + - "LOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]\n" + - "\n" + - "# Define IP address formats\n" + - "IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?\n" + - "IPV4 (? Date: Thu, 13 Oct 2016 14:08:56 -0500 Subject: [PATCH 4/7] Removed obsolete grok pattern files and removed steps in ansible scripts that create and deploy to grok patterns to HDFS directories --- .../metron_streaming/tasks/grok_upload.yml | 37 ---- .../roles/metron_streaming/tasks/main.yml | 3 - .../src/main/resources/patterns/asa | 176 ------------------ .../src/main/resources/patterns/fireeye | 9 - .../src/main/resources/patterns/sourcefire | 30 --- 5 files changed, 255 deletions(-) delete mode 100644 metron-deployment/roles/metron_streaming/tasks/grok_upload.yml delete mode 100644 metron-platform/metron-parsers/src/main/resources/patterns/asa delete mode 100644 metron-platform/metron-parsers/src/main/resources/patterns/fireeye delete mode 100644 metron-platform/metron-parsers/src/main/resources/patterns/sourcefire diff --git a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml b/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml deleted file mode 100644 index d857bf5d2c..0000000000 --- a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create HDFS directory for grok patterns - command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Assign hfds user as owner of {{ metron_hdfs_output_dir }}/patterns HDFS directory - command: hdfs dfs -chown -R hdfs:hadoop {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Assign permissions of HDFS {{ metron_hdfs_output_dir }}/patterns directory - command: hdfs dfs -chmod -R 775 {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Upload Grok Patterns to hdfs://{{ metron_hdfs_output_dir }} - command: hdfs dfs -put -f {{ metron_directory }}/patterns {{ metron_hdfs_output_dir }} - become: yes - become_user: hdfs - diff --git a/metron-deployment/roles/metron_streaming/tasks/main.yml b/metron-deployment/roles/metron_streaming/tasks/main.yml index 0945e3d6b8..63e092fc77 100644 --- a/metron-deployment/roles/metron_streaming/tasks/main.yml +++ b/metron-deployment/roles/metron_streaming/tasks/main.yml @@ -33,9 +33,6 @@ - include: hdfs_filesystem.yml run_once: true -- include: grok_upload.yml - run_once: true - - include: topologies.yml - include: source_config.yml diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/asa b/metron-platform/metron-parsers/src/main/resources/patterns/asa deleted file mode 100644 index 8c2da93e6b..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/asa +++ /dev/null @@ -1,176 +0,0 @@ -# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns - -USERNAME [a-zA-Z0-9._-]+ -USER %{USERNAME:UNWANTED} -INT (?:[+-]?(?:[0-9]+)) -BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) -NUMBER (?:%{BASE10NUM:UNWANTED}) -BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) -UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} - -# Networking -MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) -CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) -WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) -COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (?/(?>[\w_%!$@:.,~-]+|\\.)*)+ -#UNIXPATH (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ -URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? -URIHOST %{IPORHOST}(?::%{POSINT:port})? -# uripath comes loosely from RFC1738, but mostly from what Firefox -# doesn't turn into %XX -URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ -#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? -URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* -URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? -URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? - -# Months: January, Feb, 3, 03, 12, December -MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b -MONTHNUM (?:0?[1-9]|1[0-2]) -MONTHNUM2 (?:0[1-9]|1[0-2]) -MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) - -# Days: Monday, Tue, Thu, etc... -DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) - -# Years? -YEAR (?>\d\d){1,2} -# Time: HH:MM:SS -#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? -# I'm still on the fence about using grok to perform the time match, -# since it's probably slower. -# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? -HOUR (?:2[0123]|[01]?[0-9]) -MINUTE (?:[0-5][0-9]) -# '60' is a leap second in most time standards and thus is valid. -SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) -TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) -# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) -DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} -DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} -ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) -ISO8601_SECOND (?:%{SECOND}|60) -TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? -DATE %{DATE_US}|%{DATE_EU} -DATESTAMP %{DATE}[- ]%{TIME} -TZ (?:[PMCE][SD]T|UTC) -DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} -DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} -DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} -DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} -GREEDYDATA .* - -# Syslog Dates: Month Day HH:MM:SS -SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} -PROG (?:[\w._/%-]+) -SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? -SYSLOGHOST %{IPORHOST} -SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> -HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} - -# Shortcuts -QS %{QUOTEDSTRING:UNWANTED} - -# Log formats -SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: - -MESSAGESLOG %{SYSLOGBASE} %{DATA} - -COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) -COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} - -# Log Levels -LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) - -#== Cisco ASA == -CISCO_TAGGED_SYSLOG ^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:sysloghost})? ?:? %%{CISCOTAG:ciscotag}: -CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} -CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) - -# Common Particles -CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted -CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* -CISCO_DIRECTION Inbound|inbound|Outbound|outbound -CISCO_INTERVAL first hit|%{INT}-second interval -CISCO_XLATE_TYPE static|dynamic -# ASA-2-106001 -CISCOFW106001 : %{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} -# ASA-2-106006, ASA-2-106007, ASA-2-106010 -CISCOFW106006_106007_106010 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}(\(%{DATA:dst_fwuser}\))? (?:on interface %{DATA:interface}|due to %{CISCO_REASON:reason}) -# ASA-3-106014 -CISCOFW106014 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\) -# ASA-6-106015 -CISCOFW106015 : %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} -# ASA-1-106021 -CISCOFW106021 : %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} -# ASA-4-106023 -CISCOFW106023 : %{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group %{DATA:policy_id} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -# ASA-5-106100 -CISCOFW106100 : access-list %{WORD:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -# ASA-6-110002 -CISCOFW110002 : %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} -# ASA-6-302010 -CISCOFW302010 : %{INT:connection_count} in use, %{INT:connection_count_max} most used -# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 -CISCOFW302013_302014_302015_302016 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))? -# ASA-6-302020, ASA-6-302021 -CISCOFW302020_302021 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))? -# ASA-6-305011 -CISCOFW305011 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} -# ASA-3-313001, ASA-3-313004, ASA-3-313008 -CISCOFW313001_313004_313008 : %{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? -# ASA-4-313005 -CISCOFW313005 : %{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type}, code %{INT:err_icmp_code}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}(\(%{DATA:orig_dst_fwuser}\))? -# ASA-4-402117 -CISCOFW402117 : %{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip} -# ASA-4-402119 -CISCOFW402119 : %{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking -# ASA-4-419001 -CISCOFW419001 : %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason} -# ASA-4-419002 -CISCOFW419002 : %{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number -# ASA-4-500004 -CISCOFW500004 : %{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} -# ASA-6-602303, ASA-6-602304 -CISCOFW602303_602304 : %{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action} -# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 -CISCOFW710001_710002_710003_710005_710006 : %{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} -# ASA-6-713172 -CISCOFW713172 : Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device -# ASA-4-733100 -CISCOFW733100 : \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count} - - -# ASA-6-305012 -CISCOFW305012 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} duration %{TIME:duration} -# ASA-7-609001 -CISCOFW609001 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? -# ASA-7-609002 -CISCOFW609002 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? duration %{TIME:duration} - - -#== End Cisco ASA == \ No newline at end of file diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/fireeye b/metron-platform/metron-parsers/src/main/resources/patterns/fireeye deleted file mode 100644 index 5dc99bfa4f..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/fireeye +++ /dev/null @@ -1,9 +0,0 @@ -GREEDYDATA .* -POSINT \b(?:[1-9][0-9]*)\b -UID [0-9.]+ -DATA .*? - -FIREEYE_BASE ^<%{POSINT:syslog_pri}>fenotify-%{UID:uid}.alert: %{GREEDYDATA:syslog} -FIREEYE_MAIN <%{POSINT:syslog_pri}>fenotify-%{DATA:uid}.alert: %{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{GREEDYDATA:fedata} -#\|(.?)\|(.?)\|(.?)\|(.?)\|%{DATA:type}\|(.?)\|%{GREEDYDATA:fedata} -FIREEYE_SUB ^<%{POSINT:syslog_pri}>fenotify-%{UID:uid}.alert: .?*\|.?*\|.?*\|.?*\|.?*\|%{DATA:type}\|.?*\|%{GREEDYDATA:fedata} diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire b/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire deleted file mode 100644 index 672f68435e..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire +++ /dev/null @@ -1,30 +0,0 @@ -POSINT \b(?:[1-9][0-9]*)\b -NONNEGINT \b(?:[0-9]+)\b -WORD \b\w+\b -NOTSPACE \S+ -SPACE \s* -DATA .*? -GREEDYDATA .* -QUOTEDSTRING (?>(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) -UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} - -# Networking -MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) -CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) -WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) -COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (?\s%{ip_dst_addr}\:%{ip_dst_port} \ No newline at end of file From d18d3dfb3ac396f2454f82793cb7db3609f6e4c8 Mon Sep 17 00:00:00 2001 From: rmerriman Date: Mon, 17 Oct 2016 14:41:37 -0500 Subject: [PATCH 5/7] Incorporated feedback from PR comments. Includes: - reverted MessageParser.parse method back to original form and added MessageParser.configurationUpdated method - changed ParserBolt to detect config changes, set atomic flag and call MessageParser.configurationUpdated on next execute cycle - changed GrokParser to implement MessageParser.configurationUpdated method and only log grokPatterns when log level is set to debug - brought ParserBoltTest to 100% test coverage --- .../org/apache/metron/parsers/GrokParser.java | 21 +- .../metron/parsers/asa/GrokAsaParser.java | 3 +- .../metron/parsers/bolt/ParserBolt.java | 27 +- .../metron/parsers/bro/BasicBroParser.java | 3 +- .../apache/metron/parsers/csv/CSVParser.java | 3 +- .../parsers/fireeye/BasicFireEyeParser.java | 3 +- .../parsers/interfaces/MessageParser.java | 12 +- .../metron/parsers/ise/BasicIseParser.java | 3 +- .../metron/parsers/json/JSONMapParser.java | 3 +- .../parsers/lancope/BasicLancopeParser.java | 3 +- .../parsers/logstash/BasicLogstashParser.java | 3 +- .../paloalto/BasicPaloAltoFirewallParser.java | 3 +- .../parsers/snort/BasicSnortParser.java | 3 +- .../sourcefire/BasicSourcefireParser.java | 5 +- .../apache/metron/parsers/GrokParserTest.java | 5 +- .../metron/parsers/MessageParserTest.java | 13 +- .../metron/parsers/SampleGrokParserTest.java | 5 +- .../metron/parsers/SnortParserTest.java | 7 +- .../metron/parsers/asa/GrokAsaParserTest.java | 27 +- .../metron/parsers/bolt/ParserBoltTest.java | 278 +++++++++++------- .../parsers/bro/BasicBroParserTest.java | 27 +- .../metron/parsers/bro/BroParserTest.java | 11 +- .../metron/parsers/csv/CSVParserTest.java | 12 +- .../fireeye/BasicFireEyeParserTest.java | 19 +- .../parsers/ise/BasicIseParserTest.java | 5 +- .../parsers/json/JSONMapParserTest.java | 15 +- .../lancope/BasicLancopeParserTest.java | 5 +- .../BasicPaloAltoFirewallParserTest.java | 19 +- .../sourcefire/BasicSourcefireParserTest.java | 19 +- .../websphere/GrokWebSphereParserTest.java | 22 +- 30 files changed, 316 insertions(+), 268 deletions(-) diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java index 370adcf7a0..ca7bb61ba8 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java @@ -22,7 +22,6 @@ import oi.thekraken.grok.api.Grok; import oi.thekraken.grok.api.Match; import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.interfaces.MessageParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -38,7 +37,6 @@ import java.util.Date; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.TimeZone; public class GrokParser implements MessageParser, Serializable { @@ -130,9 +128,8 @@ public void init() { @SuppressWarnings("unchecked") @Override - public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { - if (grok == null || isGrokPatternUpdated(sensorParserConfig) || isPatternLabelUpdated(sensorParserConfig)) { - configure(sensorParserConfig.getParserConfig()); + public List parse(byte[] rawMessage) { + if (grok == null) { init(); } List messages = new ArrayList<>(); @@ -150,7 +147,7 @@ public List parse(byte[] rawMessage, SensorParserConfig sensorParser if (message.size() == 0) throw new RuntimeException("Grok statement produced a null message. Original message was: " + originalMessage + " , parsed message was: " + message + " , pattern was: " - + grokPattern); + + (LOG.isDebugEnabled() ? grokPattern : (patternLabel + " (Turn on DEBUG logging to see pattern text.)"))); message.put("original_string", originalMessage); for (String timeField : timeFields) { @@ -198,14 +195,10 @@ public boolean validate(JSONObject message) { return false; } - protected boolean isGrokPatternUpdated(SensorParserConfig sensorParserConfig) { - Map parserConfig = sensorParserConfig.getParserConfig(); - return parserConfig != null && !Objects.equals(grokPattern, parserConfig.get("grokPattern")); - } - - protected boolean isPatternLabelUpdated(SensorParserConfig sensorParserConfig) { - Map parserConfig = sensorParserConfig.getParserConfig(); - return parserConfig != null && !Objects.equals(patternLabel, parserConfig.get("patternLabel")); + @Override + public void configurationUpdated(Map parserConfig) { + configure(parserConfig); + init(); } protected void postParse(JSONObject message) {} diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java index 1bc2fe080f..4f1c8b0081 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java @@ -21,7 +21,6 @@ import oi.thekraken.grok.api.Match; import oi.thekraken.grok.api.exception.GrokException; import org.apache.commons.io.IOUtils; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; @@ -231,7 +230,7 @@ public void init() { } @Override - public List parse(byte[] raw_message, SensorParserConfig sensorParserConfig) { + public List parse(byte[] raw_message) { String toParse = ""; JSONObject toReturn; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 9e491fb58f..8df14dd224 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -25,23 +25,28 @@ import backtype.storm.tuple.Values; import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredParserBolt; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.FieldValidator; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.FunctionResolver; import org.apache.metron.common.dsl.StellarFunctions; +import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.parsers.filters.Filters; -import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.parsers.filters.GenericMessageFilter; -import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.parsers.interfaces.MessageFilter; import org.apache.metron.parsers.interfaces.MessageParser; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; public class ParserBolt extends ConfiguredParserBolt implements Serializable { @@ -51,6 +56,7 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable { private MessageFilter filter = new GenericMessageFilter(); private WriterHandler writer; private org.apache.metron.common.dsl.Context stellarContext; + private AtomicBoolean configUpdatedFlag = new AtomicBoolean(false); public ParserBolt( String zookeeperUrl , String sensorType , MessageParser parser @@ -116,8 +122,11 @@ public void execute(Tuple tuple) { boolean ackTuple = !writer.handleAck(); int numWritten = 0; if(sensorParserConfig != null) { + if (configUpdatedFlag.getAndSet(false)) { + parser.configurationUpdated(getSensorParserConfig().getParserConfig()); + } List fieldValidations = getConfigurations().getFieldValidations(); - Optional> messages = parser.parseOptional(originalMessage, sensorParserConfig); + Optional> messages = parser.parseOptional(originalMessage); for (JSONObject message : messages.orElse(Collections.emptyList())) { message.put(Constants.SENSOR_TYPE, getSensorType()); for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) { @@ -168,4 +177,12 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declareStream(Constants.INVALID_STREAM, new Fields("message")); declarer.declareStream(Constants.ERROR_STREAM, new Fields("message")); } + + @Override + public void updateConfig(String path, byte[] data) throws IOException { + super.updateConfig(path, data); + if (path.startsWith(ConfigurationType.PARSER.getZookeeperRoot() + "/" + getSensorType())) { + configUpdatedFlag.set(true); + } + } } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java index b8e7587ede..df522050ba 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java @@ -19,7 +19,6 @@ package org.apache.metron.parsers.bro; import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -55,7 +54,7 @@ public void init() { } @SuppressWarnings("unchecked") - public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { + public List parse(byte[] msg) { _LOG.trace("[Metron] Starting to parse incoming message"); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java index e44c831b8c..52d45c9118 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java @@ -19,7 +19,6 @@ package org.apache.metron.parsers.csv; import com.google.common.collect.ImmutableList; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.csv.CSVConverter; import org.apache.metron.common.utils.ConversionUtils; import org.apache.metron.parsers.BasicParser; @@ -54,7 +53,7 @@ public void init() { @Override - public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { + public List parse(byte[] rawMessage) { try { String msg = new String(rawMessage, "UTF-8"); Map value = converter.toMap(msg); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java index 80041821e6..498248a517 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java @@ -21,7 +21,6 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import org.apache.commons.lang3.StringUtils; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.utils.ParserUtils; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; @@ -71,7 +70,7 @@ public void init() { } @Override - public List parse(byte[] raw_message, SensorParserConfig sensorParserConfig) { + public List parse(byte[] raw_message) { String toParse = ""; List messages = new ArrayList<>(); try { diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java index 2af701b5c4..9157b75cc2 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java @@ -17,9 +17,8 @@ */ package org.apache.metron.parsers.interfaces; -import org.apache.metron.common.configuration.SensorParserConfig; - import java.util.List; +import java.util.Map; import java.util.Optional; public interface MessageParser extends Configurable { @@ -34,15 +33,15 @@ public interface MessageParser extends Configurable { * @param rawMessage * @return If null is returned, this is treated as an empty list. */ - List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig); + List parse(byte[] rawMessage); /** * Take raw data and convert it to an optional list of messages. * @param parseMessage * @return If null is returned, this is treated as an empty list. */ - default Optional> parseOptional(byte[] parseMessage, SensorParserConfig sensorParserConfig) { - return Optional.ofNullable(parse(parseMessage, sensorParserConfig)); + default Optional> parseOptional(byte[] parseMessage) { + return Optional.ofNullable(parse(parseMessage)); } /** @@ -52,4 +51,7 @@ default Optional> parseOptional(byte[] parseMessage, SensorParserConfig */ boolean validate(T message); + default void configurationUpdated(Map config) { + } + } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java index 7463414e8f..2d559ac6cf 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java @@ -20,7 +20,6 @@ package org.apache.metron.parsers.ise; import com.esotericsoftware.minlog.Log; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -50,7 +49,7 @@ public void init() { @SuppressWarnings("unchecked") @Override - public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { + public List parse(byte[] msg) { String raw_message = ""; List messages = new ArrayList<>(); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java index c3103ae8c8..5d824561dc 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; @@ -93,7 +92,7 @@ public void init() { * @return If null is returned, this is treated as an empty list. */ @Override - public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { + public List parse(byte[] rawMessage) { try { String originalString = new String(rawMessage); //convert the JSON blob into a String -> Object map diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java index 36c8c3a5bb..83eedcc7f9 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java @@ -18,7 +18,6 @@ package org.apache.metron.parsers.lancope; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.json.simple.JSONValue; @@ -51,7 +50,7 @@ public void init() { //@SuppressWarnings("unchecked") @Override - public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { + public List parse(byte[] msg) { JSONObject payload = null; List messages = new ArrayList<>(); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java index c220fde30f..2f5310c9d6 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java @@ -17,7 +17,6 @@ */ package org.apache.metron.parsers.logstash; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -40,7 +39,7 @@ public void init() { } @Override - public List parse(byte[] raw_message, SensorParserConfig sensorParserConfig) { + public List parse(byte[] raw_message) { List messages = new ArrayList<>(); try { diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java index d5615189bc..e6b927414d 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java @@ -18,7 +18,6 @@ package org.apache.metron.parsers.paloalto; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -103,7 +102,7 @@ public void init() { } @SuppressWarnings({"unchecked", "unused"}) - public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { + public List parse(byte[] msg) { JSONObject outputMessage = new JSONObject(); String toParse = ""; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java index 0e2a122c21..e50926cdc6 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java @@ -21,7 +21,6 @@ import jdk.nashorn.internal.runtime.arrays.ArrayIndex; import org.apache.metron.common.Constants; import org.apache.metron.common.csv.CSVConverter; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -99,7 +98,7 @@ public void init() { } @Override - public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { + public List parse(byte[] rawMessage) { JSONObject jsonMessage = new JSONObject(); List messages = new ArrayList<>(); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java index b57995d070..28a298378c 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java @@ -18,7 +18,6 @@ package org.apache.metron.parsers.sourcefire; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -40,7 +39,7 @@ public class BasicSourcefireParser extends BasicParser { String domain_name_regex = "([^\\.]+)\\.([a-z]{2}|[a-z]{3}|([a-z]{2}\\.[a-z]{2}))$"; String sidRegex = "(.*)(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; //String sidRegex = "(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; - Pattern sidPattern = Pattern.compile(sidRegex); + Pattern sidPattern = Pattern.compile(sidRegex); Pattern pattern = Pattern.compile(domain_name_regex); @Override @@ -54,7 +53,7 @@ public void init() { } @SuppressWarnings({ "unchecked", "unused" }) - public List parse(byte[] msg, SensorParserConfig sensorParserConfig) { + public List parse(byte[] msg) { JSONObject payload = new JSONObject(); String toParse = ""; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java index 1275fc7a4a..e4a3f437ad 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java @@ -20,7 +20,6 @@ import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import junit.framework.Assert; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -43,8 +42,6 @@ public void test() throws IOException, ParseException { parserConfig.put("timestampField", getTimestampField()); parserConfig.put("dateFormat", getDateFormat()); parserConfig.put("timeFields", getTimeFields()); - SensorParserConfig sensorParserConfig = new SensorParserConfig(); - sensorParserConfig.setParserConfig(parserConfig); GrokParser grokParser = new GrokParser(); grokParser.configure(parserConfig); @@ -57,7 +54,7 @@ public void test() throws IOException, ParseException { JSONObject expected = (JSONObject) jsonParser.parse(e.getValue()); byte[] rawMessage = e.getKey().getBytes(); - List parsedList = grokParser.parse(rawMessage, sensorParserConfig); + List parsedList = grokParser.parse(rawMessage); Assert.assertEquals(1, parsedList.size()); compare(expected, parsedList.get(0)); } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java index 0888dfe44c..1d2af78615 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java @@ -19,7 +19,6 @@ package org.apache.metron.parsers; import junit.framework.Assert; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.interfaces.MessageParser; import org.junit.Test; @@ -38,7 +37,7 @@ public void init() { } @Override - public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { + public List parse(byte[] rawMessage) { return null; } @@ -52,8 +51,8 @@ public void configure(Map config) { } }; - Assert.assertNotNull(parser.parseOptional(null, null)); - Assert.assertFalse(parser.parseOptional(null, null).isPresent()); + Assert.assertNotNull(parser.parseOptional(null)); + Assert.assertFalse(parser.parseOptional(null).isPresent()); } @Test @@ -65,7 +64,7 @@ public void init() { } @Override - public List parse(byte[] rawMessage, SensorParserConfig sensorParserConfig) { + public List parse(byte[] rawMessage) { return new ArrayList<>(); } @@ -79,8 +78,8 @@ public void configure(Map config) { } }; - Assert.assertNotNull(parser.parseOptional(null, null)); - Optional ret = parser.parseOptional(null, null); + Assert.assertNotNull(parser.parseOptional(null)); + Optional ret = parser.parseOptional(null); Assert.assertTrue(ret.isPresent()); Assert.assertEquals(0, ret.get().size()); } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java index 6266985a5b..90f2d4b311 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java @@ -118,12 +118,13 @@ public void testConfigChange() { grokParser.configure(parserConfig); grokParser.init(); - List results = grokParser.parse(raw.getBytes(), sensorParserConfig); + List results = grokParser.parse(raw.getBytes()); Assert.assertEquals(1, results.size()); compare(expected1, results.get(0)); parserConfig.put("grokPattern", pattern2); - results = grokParser.parse(raw.getBytes(), sensorParserConfig); + grokParser.configurationUpdated(sensorParserConfig.getParserConfig()); + results = grokParser.parse(raw.getBytes()); Assert.assertEquals(1, results.size()); compare(expected2, results.get(0)); } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java index e621abb070..8114e836df 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java @@ -19,7 +19,6 @@ package org.apache.metron.parsers; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.snort.BasicSnortParser; import org.junit.Assert; import org.junit.Test; @@ -33,14 +32,12 @@ public class SnortParserTest { @Multiline public static String goodMessage; - private SensorParserConfig sensorParserConfig = new SensorParserConfig(); - @Test public void testGoodMessage() { BasicSnortParser parser = new BasicSnortParser(); parser.init(); - Map out = parser.parse(goodMessage.getBytes(), sensorParserConfig).get(0); + Map out = parser.parse(goodMessage.getBytes()).get(0); Assert.assertEquals(out.get("msg"),"Consecutive TCP small segments, exceeding threshold"); Assert.assertEquals(out.get("sig_rev"), "1"); Assert.assertEquals(out.get("ip_dst_addr"), "10.0.2.15"); @@ -74,6 +71,6 @@ public void testGoodMessage() { public void testBadMessage() { BasicSnortParser parser = new BasicSnortParser(); parser.init(); - parser.parse("foo bar".getBytes(), sensorParserConfig); + parser.parse("foo bar".getBytes()); } } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java index fb3fdce952..39c895f689 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java @@ -20,7 +20,6 @@ import java.util.Iterator; import java.util.Map; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.sourcefire.BasicSourcefireParser; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -44,21 +43,21 @@ public class GrokAsaParserTest extends AbstractConfigTest{ * The grokAsaStrings. */ private static String[] grokAsaStrings=null; - + /** * The grokAsaParser. */ - + private GrokAsaParser grokAsaParser=null; - + /** * Constructs a new GrokAsaParserTest instance. * @throws Exception */ - + public GrokAsaParserTest() throws Exception { - super(); - + super(); + } /** * @throws java.lang.Exception @@ -73,19 +72,19 @@ public static void tearDownAfterClass() throws Exception { setGrokAsaStrings(null); } - /* + /* * (non-Javadoc) * @see junit.framework.TestCase#setUp() */ public void setUp() throws Exception { super.setUp("org.apache.metron.parsers.asa.GrokAsaParserTest"); setGrokAsaStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - grokAsaParser = new GrokAsaParser(); + grokAsaParser = new GrokAsaParser(); } /** - * - * + * + * * @throws java.lang.Exception */ public void tearDown() throws Exception { @@ -93,13 +92,13 @@ public void tearDown() throws Exception { } /** - * Test method for {@link BasicSourcefireParser#parse(byte[], SensorParserConfig)}. + * Test method for {@link BasicSourcefireParser#parse(byte[])}. */ @SuppressWarnings({ "rawtypes" }) public void testParse() { - + for (String grokAsaString : getGrokAsaStrings()) { - JSONObject parsed = grokAsaParser.parse(grokAsaString.getBytes(), new SensorParserConfig()).get(0); + JSONObject parsed = grokAsaParser.parse(grokAsaString.getBytes()).get(0); Assert.assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index a4492e1c34..1dcd238cf9 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -17,28 +17,26 @@ */ package org.apache.metron.parsers.bolt; -import org.apache.metron.common.configuration.SensorParserConfig; - import backtype.storm.task.OutputCollector; import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; import com.google.common.collect.ImmutableList; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ParserConfigurations; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.dsl.Context; import org.apache.metron.common.writer.BulkMessageWriter; -import org.adrianwalker.multilinestring.Multiline; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.metron.common.configuration.ParserConfigurations; -import org.apache.metron.common.configuration.SensorParserConfig; -import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.common.writer.BulkWriterResponse; +import org.apache.metron.common.writer.MessageWriter; import org.apache.metron.parsers.BasicParser; -import org.apache.metron.parsers.csv.CSVParser; -import org.apache.metron.test.bolt.BaseBoltTest; import org.apache.metron.parsers.interfaces.MessageFilter; import org.apache.metron.parsers.interfaces.MessageParser; -import org.apache.metron.common.writer.MessageWriter; +import org.apache.metron.test.bolt.BaseBoltTest; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.junit.Assert; @@ -46,8 +44,13 @@ import org.mockito.Mock; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; @@ -114,20 +117,19 @@ public List getRecords() { @Test public void testEmpty() throws Exception { String sensorType = "yaf"; - SensorParserConfig sensorParserConfig = new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - }}; - } - }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return sensorParserConfig; + return new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + }}; + } + }; } }; } @@ -141,41 +143,105 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { byte[] sampleBinary = "some binary message".getBytes(); when(tuple.getBinary(0)).thenReturn(sampleBinary); - when(parser.parseOptional(sampleBinary, sensorParserConfig)).thenReturn(null); + when(parser.parseOptional(sampleBinary)).thenReturn(null); parserBolt.execute(tuple); verify(parser, times(0)).validate(any()); verify(writer, times(0)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), any()); verify(outputCollector, times(1)).ack(tuple); } - @Test - public void test() throws Exception { + /** + { + "fieldValidations" : [ + { + "input" : [ "field1" ], + "validation" : "NOT_EMPTY" + } + ] + } + */ + @Multiline + public static String globalConfig; + + + /** + { + "sensorTopic": "yaf", + "filterClassName": "org.apache.metron.parsers.filters.GenericMessageFilter", + "parserConfig" : { + "config" : "value" + }, + "fieldTransformations" : [ + { + "transformation" : "STELLAR" + ,"output" : [ "field3" ] + ,"config" : { + "field3" : "TO_UPPER(field1)" + } + } + ] + } + */ + @Multiline + public static String sensorParserConfig; + /** + { + "sensorTopic": "yaf", + "parserConfig" : { + "config" : "updatedValue" + } + } + */ + @Multiline + public static String updatedParserConfig; + + @Test + public void testPrepare() throws Exception { String sensorType = "yaf"; - SensorParserConfig sensorParserConfig = new SensorParserConfig() { + ParserConfigurations parserConfigurations = new ParserConfigurations(); + ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) { @Override - public Map getParserConfig() { - return new HashMap() {{ - }}; + protected ParserConfigurations defaultConfigurations() { + return parserConfigurations; } }; + parserBolt.setCuratorFramework(client); + parserBolt.setTreeCache(cache); + try { + parserBolt.prepare(new HashMap(), topologyContext, outputCollector); + fail("ParserBolt.prepare should throw exception on empty config"); + } catch(IllegalStateException e) { + } + + parserConfigurations.updateSensorParserConfig(sensorType, Bytes.toBytes(sensorParserConfig)); + parserBolt.withMessageFilter(null); + parserBolt.prepare(new HashMap(), topologyContext, outputCollector); + verify(parser, times(2)).init(); + verify(writer, times(2)).init(); + } + + @Test + public void test() throws Exception { + + String sensorType = "yaf"; + + ParserConfigurations parserConfigurations = new ParserConfigurations(); + parserConfigurations.updateGlobalConfig(Bytes.toBytes(globalConfig)); + parserConfigurations.updateSensorParserConfig(sensorType, Bytes.toBytes(sensorParserConfig)); ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) { @Override protected ParserConfigurations defaultConfigurations() { - return new ParserConfigurations() { - @Override - public SensorParserConfig getSensorParserConfig(String sensorType) { - return sensorParserConfig; - } - }; + return parserConfigurations; } }; parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); - verify(parser, times(1)).init(); - verify(writer, times(1)).init(); + parserBolt.declareOutputFields(declarer); + verify(declarer, times(1)).declareStream(eq(Constants.INVALID_STREAM), any()); + verify(declarer, times(1)).declareStream(eq(Constants.ERROR_STREAM), any()); byte[] sampleBinary = "some binary message".getBytes(); JSONParser jsonParser = new JSONParser(); final JSONObject sampleMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\" }"); @@ -184,24 +250,31 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { add(sampleMessage1); add(sampleMessage2); }}; - final JSONObject finalMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"source.type\":\"" + sensorType + "\" }"); - final JSONObject finalMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"source.type\":\"" + sensorType + "\" }"); + final JSONObject finalMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"field3\":\"VALUE1\", \"source.type\":\"" + sensorType + "\" }"); + final JSONObject finalMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"source.type\":\"" + sensorType + ".invalid\" }"); when(tuple.getBinary(0)).thenReturn(sampleBinary); - when(parser.parseOptional(sampleBinary, sensorParserConfig)).thenReturn(Optional.of(messages)); + when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messages)); when(parser.validate(eq(messages.get(0)))).thenReturn(true); - when(parser.validate(eq(messages.get(1)))).thenReturn(false); + when(parser.validate(eq(messages.get(1)))).thenReturn(true); parserBolt.execute(tuple); verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1)); verify(outputCollector, times(1)).ack(tuple); - when(parser.validate(eq(messages.get(0)))).thenReturn(true); - when(parser.validate(eq(messages.get(1)))).thenReturn(true); - when(filter.emitTuple(eq(messages.get(0)), any())).thenReturn(false); - when(filter.emitTuple(eq(messages.get(1)), any())).thenReturn(true); - parserBolt.withMessageFilter(filter); + verify(parser, times(0)).configurationUpdated(any()); + verify(outputCollector, times(1)).emit(eq(Constants.INVALID_STREAM), eq(new Values(finalMessage2))); + + parserBolt.updateConfig(ConfigurationType.PARSER.getZookeeperRoot() + "/" + sensorType, Bytes.toBytes(updatedParserConfig)); parserBolt.execute(tuple); - verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2)); + verify(writer, times(2)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1)); verify(outputCollector, times(2)).ack(tuple); - doThrow(new Exception()).when(writer).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2)); + verify(parser, times(1)).configurationUpdated(SensorParserConfig.fromBytes(Bytes.toBytes(updatedParserConfig)).getParserConfig()); + + when(filter.emitTuple(eq(messages.get(0)), any())).thenReturn(true); + when(filter.emitTuple(eq(messages.get(1)), any())).thenReturn(false); + parserBolt.withMessageFilter(filter); + parserBolt.execute(tuple); + verify(writer, times(3)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1)); + verify(outputCollector, times(3)).ack(tuple); + doThrow(new Exception()).when(writer).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1)); parserBolt.execute(tuple); verify(outputCollector, times(1)).reportError(any(Throwable.class)); } @@ -210,20 +283,20 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { public void testImplicitBatchOfOne() throws Exception { String sensorType = "yaf"; - SensorParserConfig sensorParserConfig = new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - }}; - } - }; + ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return sensorParserConfig; + return new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + }}; + } + }; } }; } @@ -234,38 +307,36 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { verify(parser, times(1)).init(); verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any(), eq(sensorParserConfig))).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); verify(outputCollector, times(1)).ack(t1); + verify(parser, times(0)).configurationUpdated(any()); } + /** + { + "filterClassName" : "QUERY" + ,"parserConfig" : { + "filter.query" : "exists(field1)" + } + } + */ + @Multiline + public static String filterSensorParserConfig; @Test public void testFilter() throws Exception { String sensorType = "yaf"; - SensorParserConfig sensorParserConfig = new SensorParserConfig() { - @Override - public String getFilterClassName() { - return "QUERY"; - } - @Override - public Map getParserConfig() { - return new HashMap() {{ - put("filter.query", "exists(field1)"); - }}; - } - }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override - protected ParserConfigurations defaultConfigurations() { - return new ParserConfigurations() { - @Override - public SensorParserConfig getSensorParserConfig(String sensorType) { - return sensorParserConfig; - } - }; + protected SensorParserConfig getSensorParserConfig() { + try { + return SensorParserConfig.fromBytes(Bytes.toBytes(filterSensorParserConfig)); + } catch (IOException e) { + throw new RuntimeException(e); + } } }; parserBolt.setCuratorFramework(client); @@ -274,7 +345,7 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { verify(parser, times(1)).init(); verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any(), eq(sensorParserConfig))).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); verify(outputCollector, times(1)).ack(t1); @@ -351,21 +422,21 @@ protected SensorParserConfig getSensorParserConfig() { public void testBatchOfOne() throws Exception { String sensorType = "yaf"; - SensorParserConfig sensorParserConfig = new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - put(ParserWriterConfiguration.BATCH_CONF, "1"); - }}; - } - }; + ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return sensorParserConfig; + return new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + put(ParserWriterConfiguration.BATCH_CONF, "1"); + }}; + } + }; } }; } @@ -376,7 +447,7 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { verify(parser, times(1)).init(); verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any(), eq(sensorParserConfig))).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); @@ -386,21 +457,21 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { public void testBatchOfFive() throws Exception { String sensorType = "yaf"; - SensorParserConfig sensorParserConfig = new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - put(ParserWriterConfiguration.BATCH_CONF, 5); - }}; - } - }; + ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return sensorParserConfig; + return new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + put(ParserWriterConfiguration.BATCH_CONF, 5); + }}; + } + }; } }; } @@ -411,7 +482,7 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { verify(parser, times(1)).init(); verify(batchWriter, times(1)).init(any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any(), eq(sensorParserConfig))).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); writeNonBatch(outputCollector, parserBolt, t1); @@ -431,21 +502,20 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { public void testBatchOfFiveWithError() throws Exception { String sensorType = "yaf"; - SensorParserConfig sensorParserConfig = new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - put(ParserWriterConfiguration.BATCH_CONF, 5); - }}; - } - }; ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { @Override protected ParserConfigurations defaultConfigurations() { return new ParserConfigurations() { @Override public SensorParserConfig getSensorParserConfig(String sensorType) { - return sensorParserConfig; + return new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + put(ParserWriterConfiguration.BATCH_CONF, 5); + }}; + } + }; } }; } @@ -458,7 +528,7 @@ public SensorParserConfig getSensorParserConfig(String sensorType) { doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any()); when(parser.validate(any())).thenReturn(true); - when(parser.parse(any(), eq(sensorParserConfig))).thenReturn(ImmutableList.of(new JSONObject())); + when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject())); when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); parserBolt.withMessageFilter(filter); parserBolt.execute(t1); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java index 3a5aa8efbb..55c6695af0 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java @@ -19,7 +19,6 @@ import junit.framework.TestCase; import org.apache.commons.lang3.tuple.Pair; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -35,7 +34,6 @@ public class BasicBroParserTest extends TestCase { */ private BasicBroParser broParser = null; private JSONParser jsonParser = null; - private SensorParserConfig sensorParserConfig = null; /** * Constructs a new BasicBroParserTest instance. @@ -45,7 +43,6 @@ public class BasicBroParserTest extends TestCase { public BasicBroParserTest() throws Exception { broParser = new BasicBroParser(); jsonParser = new JSONParser(); - sensorParserConfig = new SensorParserConfig(); } /** @@ -73,7 +70,7 @@ public void testUnwrappedBroMessage() throws ParseException { JSONObject rawJson = (JSONObject)jsonParser.parse(rawMessage); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedBroTimestamp = "1449511228.474"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); @@ -99,7 +96,7 @@ public void testHttpBroMessage() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedBroTimestamp = "1402307733.473"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1402307733473"; @@ -130,7 +127,7 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedTimestamp = "1467657279000"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); String expectedBroTimestamp = "1467657279.0"; @@ -143,7 +140,7 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedTimestamp = "1467657279000"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); String expectedBroTimestamp = "1467657279.0"; @@ -156,7 +153,7 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedTimestamp = "1467657279100"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); String expectedBroTimestamp = "1467657279.1"; @@ -169,7 +166,7 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedTimestamp = "1467657279110"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); String expectedBroTimestamp = "1467657279.11"; @@ -183,7 +180,7 @@ public void testHttpDecimalBroMessage() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedBroTimestamp = "1457149494.166991"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1457149494166"; @@ -208,7 +205,7 @@ public void testDnsBroMessage() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedBroTimestamp = "1402308259.609"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1402308259609"; @@ -230,7 +227,7 @@ public void testFilesBroMessage() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedBroTimestamp = "1425845251.334"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1425845251334"; @@ -251,7 +248,7 @@ public void testProtocolKeyCleanedUp() throws ParseException { Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes(), sensorParserConfig).get(0); + JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); String expectedBroTimestamp = "1402307733.473"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1402307733473"; @@ -262,7 +259,7 @@ public void testProtocolKeyCleanedUp() throws ParseException { public void testBadMessage() throws ParseException{ try { - broParser.parse("{ \"foo\" : \"bar\"}".getBytes(), sensorParserConfig); + broParser.parse("{ \"foo\" : \"bar\"}".getBytes()); Assert.fail("Should have marked this as a bad message."); } catch(IllegalStateException ise) { @@ -270,7 +267,7 @@ public void testBadMessage() throws ParseException{ } //non json try { - broParser.parse("foo bar".getBytes(), sensorParserConfig); + broParser.parse("foo bar".getBytes()); Assert.fail("Should have marked this as a bad message."); } catch(IllegalStateException ise) { diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java index b7a3b99280..9747f19120 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java @@ -24,7 +24,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -51,8 +50,8 @@ * @version $Revision: 1.1 $ */ public class BroParserTest extends AbstractConfigTest { - - + + /** * The inputStrings. */ @@ -90,9 +89,9 @@ public static void tearDownAfterClass() throws Exception { public void setUp() throws Exception { super.setUp("org.apache.metron.parsers.bro.BroParserTest"); setInputStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - parser = new BasicBroParser(); + parser = new BasicBroParser(); } - + /** * @throws ParseException * Tests for Parse Method @@ -102,7 +101,7 @@ public void setUp() throws Exception { public void testParse() throws ParseException { for (String inputString : getInputStrings()) { - JSONObject cleanJson = parser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); + JSONObject cleanJson = parser.parse(inputString.getBytes()).get(0); Assert.assertNotNull(cleanJson); System.out.println(cleanJson); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java index cfa5d4d1d4..e667e54645 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java @@ -56,15 +56,15 @@ public void test() throws IOException { parser.configure(config.getParserConfig()); { String line = "#foo,bar,grok"; - Assert.assertEquals(0, parser.parse(Bytes.toBytes(line), config).size()); + Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size()); } { String line = ""; - Assert.assertEquals(0, parser.parse(Bytes.toBytes(line), config).size()); + Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size()); } { String line = "foo,bar,grok"; - List results = parser.parse(Bytes.toBytes(line), config); + List results = parser.parse(Bytes.toBytes(line)); Assert.assertEquals(1, results.size()); JSONObject o = results.get(0); Assert.assertTrue(parser.validate(o)); @@ -75,7 +75,7 @@ public void test() throws IOException { } { String line = "\"foo\", \"bar\",\"grok\""; - List results = parser.parse(Bytes.toBytes(line), config); + List results = parser.parse(Bytes.toBytes(line)); Assert.assertEquals(1, results.size()); JSONObject o = results.get(0); Assert.assertTrue(parser.validate(o)); @@ -86,7 +86,7 @@ public void test() throws IOException { } { String line = "foo, bar, grok"; - List results = parser.parse(Bytes.toBytes(line), config); + List results = parser.parse(Bytes.toBytes(line)); Assert.assertEquals(1, results.size()); JSONObject o = results.get(0); Assert.assertTrue(parser.validate(o)); @@ -98,7 +98,7 @@ public void test() throws IOException { { String line = "foo"; try { - List results = parser.parse(Bytes.toBytes(line), config); + List results = parser.parse(Bytes.toBytes(line)); Assert.fail("Expected exception"); } catch(IllegalStateException iae) {} diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java index 006693ed1e..a00dcd51f5 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java @@ -22,7 +22,6 @@ import java.util.Iterator; import java.util.Map; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -44,17 +43,17 @@ public class BasicFireEyeParserTest extends AbstractConfigTest * The inputStrings. */ private static String[] inputStrings; - + /** * The parser. */ private BasicFireEyeParser parser=null; - + /** * Constructs a new BasicFireEyeParserTest instance. * @throws Exception - */ + */ public BasicFireEyeParserTest() throws Exception { super(); } @@ -78,17 +77,17 @@ public static void tearDownAfterClass() throws Exception { public void setUp() throws Exception { super.setUp("org.apache.metron.parsers.fireeye.BasicFireEyeParserTest"); setInputStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - parser = new BasicFireEyeParser(); + parser = new BasicFireEyeParser(); } /** - * - * + * + * * @throws java.lang.Exception */ public void tearDown() throws Exception { parser = null; - setInputStrings(null); + setInputStrings(null); } /** @@ -98,12 +97,12 @@ public void tearDown() throws Exception { * * * - * {@link BasicFireEyeParser#parse(byte[], SensorParserConfig)}. + * {@link BasicFireEyeParser#parse(byte[])}. */ @SuppressWarnings({ "rawtypes"}) public void testParse() { for (String inputString : getInputStrings()) { - JSONObject parsed = parser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); + JSONObject parsed = parser.parse(inputString.getBytes()).get(0); Assert.assertNotNull(parsed); JSONParser parser = new JSONParser(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java index cc084d59d5..cac64e9ebf 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java @@ -21,7 +21,6 @@ import java.net.URL; import java.util.Map; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -104,14 +103,14 @@ protected void tearDown() throws Exception { /** * Test method for - * {@link BasicIseParser#parse(byte[], SensorParserConfig)}. + * {@link BasicIseParser#parse(byte[])}. * * @throws IOException * @throws Exception */ public void testParse() throws ParseException, IOException, Exception { for (String inputString : getInputStrings()) { - JSONObject parsed = parser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); + JSONObject parsed = parser.parse(inputString.getBytes()).get(0); assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java index 6006f6511d..61748e7e14 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; @@ -42,7 +41,7 @@ public class JSONMapParserTest { @Test public void testHappyPath() { JSONMapParser parser = new JSONMapParser(); - List output = parser.parse(happyPathJSON.getBytes(), sensorParserConfig); + List output = parser.parse(happyPathJSON.getBytes()); Assert.assertEquals(output.size(), 1); //don't forget the timestamp field! Assert.assertEquals(output.get(0).size(), 5); @@ -74,12 +73,10 @@ public void testHappyPath() { @Multiline static String mixCollectionHandlingJSON; - private SensorParserConfig sensorParserConfig = new SensorParserConfig(); - @Test public void testCollectionHandlingDrop() { JSONMapParser parser = new JSONMapParser(); - List output = parser.parse(collectionHandlingJSON.getBytes(), sensorParserConfig); + List output = parser.parse(collectionHandlingJSON.getBytes()); Assert.assertEquals(output.size(), 1); //don't forget the timestamp field! Assert.assertEquals(output.get(0).size(), 2); @@ -92,7 +89,7 @@ public void testCollectionHandlingDrop() { public void testCollectionHandlingError() { JSONMapParser parser = new JSONMapParser(); parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ERROR.name())); - parser.parse(collectionHandlingJSON.getBytes(), sensorParserConfig); + parser.parse(collectionHandlingJSON.getBytes()); } @@ -100,7 +97,7 @@ public void testCollectionHandlingError() { public void testCollectionHandlingAllow() { JSONMapParser parser = new JSONMapParser(); parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ALLOW.name())); - List output = parser.parse(collectionHandlingJSON.getBytes(), sensorParserConfig); + List output = parser.parse(collectionHandlingJSON.getBytes()); Assert.assertEquals(output.size(), 1); //don't forget the timestamp field! Assert.assertEquals(output.get(0).size(), 3); @@ -113,7 +110,7 @@ public void testCollectionHandlingAllow() { public void testCollectionHandlingUnfold() { JSONMapParser parser = new JSONMapParser(); parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.UNFOLD.name())); - List output = parser.parse(collectionHandlingJSON.getBytes(), sensorParserConfig); + List output = parser.parse(collectionHandlingJSON.getBytes()); Assert.assertEquals(output.size(), 1); //don't forget the timestamp field! Assert.assertEquals(output.get(0).size(), 6); @@ -130,7 +127,7 @@ public void testCollectionHandlingUnfold() { public void testMixedCollectionHandlingUnfold() { JSONMapParser parser = new JSONMapParser(); parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG,JSONMapParser.MapStrategy.UNFOLD.name())); - List output = parser.parse(mixCollectionHandlingJSON.getBytes(), sensorParserConfig); + List output = parser.parse(mixCollectionHandlingJSON.getBytes()); Assert.assertEquals(output.get(0).size(), 4); JSONObject message = output.get(0); Assert.assertEquals(message.get("collection.key"), "value"); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java index ba4638676e..0d2bcb331d 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java @@ -21,7 +21,6 @@ import java.net.URL; import java.util.Map; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -98,14 +97,14 @@ protected void tearDown() throws Exception { } /** - * Test method for {@link BasicLancopeParser#parse(byte[], SensorParserConfig)}. + * Test method for {@link BasicLancopeParser#parse(byte[])}. * @throws Exception * @throws IOException */ public void testParse() throws IOException, Exception { for (String inputString : getInputStrings()) { - JSONObject parsed = parser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); + JSONObject parsed = parser.parse(inputString.getBytes()).get(0); assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java index ac9c89ca89..5dd27fcb1d 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java @@ -20,7 +20,6 @@ import java.util.Iterator; import java.util.Map; -import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.parsers.sourcefire.BasicSourcefireParser; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -38,18 +37,18 @@ public class BasicPaloAltoFirewallParserTest extends AbstractConfigTest { /** * Constructs a new BasicPaloAltoFirewallParserTest instance. * @throws Exception - */ + */ public BasicPaloAltoFirewallParserTest() throws Exception { - super(); + super(); } /** * Sets the inputStrings. * @param inputStrings the inputStrings. */ - + public static void setInputStrings(String[] inputStrings) { - + BasicPaloAltoFirewallParserTest.inputStrings = inputStrings; } @@ -77,12 +76,12 @@ public static void tearDownAfterClass() throws Exception { public void setUp() throws Exception { super.setUp("org.apache.metron.parsers.paloalto.BasicPaloAltoFirewallParserTest"); setPAStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - paParser = new BasicPaloAltoFirewallParser(); + paParser = new BasicPaloAltoFirewallParser(); } /** - * - * + * + * * @throws java.lang.Exception */ public void tearDown() throws Exception { @@ -91,12 +90,12 @@ public void tearDown() throws Exception { /** * Test method for - * {@link BasicSourcefireParser#parse(byte[], SensorParserConfig)}. + * {@link BasicSourcefireParser#parse(byte[])}. */ @SuppressWarnings({ "rawtypes" }) public void testParse() { for (String inputString : getInputStrings()) { - JSONObject parsed = paParser.parse(inputString.getBytes(), new SensorParserConfig()).get(0); + JSONObject parsed = paParser.parse(inputString.getBytes()).get(0); Assert.assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java index f032c6d27f..03a7f6e1bc 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java @@ -22,7 +22,6 @@ import java.util.Iterator; import java.util.Map; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -42,9 +41,9 @@ public class BasicSourcefireParserTest extends AbstractConfigTest { /** * The sourceFireStrings. - */ + */ private static String[] sourceFireStrings; - + /** * The sourceFireParser. */ @@ -55,11 +54,11 @@ public class BasicSourcefireParserTest extends AbstractConfigTest * Constructs a new BasicSourcefireParserTest instance. * @throws Exception */ - + public BasicSourcefireParserTest() throws Exception { - super(); + super(); } - + /** * @throws java.lang.Exception */ @@ -83,8 +82,8 @@ public void setUp() throws Exception { } /** - * - * + * + * * @throws java.lang.Exception */ public void tearDown() throws Exception { @@ -92,13 +91,13 @@ public void tearDown() throws Exception { } /** - * Test method for {@link BasicSourcefireParser#parse(byte[], SensorParserConfig)}. + * Test method for {@link BasicSourcefireParser#parse(byte[])}. */ @SuppressWarnings({ "rawtypes", "unused" }) public void testParse() { for (String sourceFireString : getSourceFireStrings()) { byte[] srcBytes = sourceFireString.getBytes(); - JSONObject parsed = sourceFireParser.parse(sourceFireString.getBytes(), new SensorParserConfig()).get(0); + JSONObject parsed = sourceFireParser.parse(sourceFireString.getBytes()).get(0); Assert.assertNotNull(parsed); System.out.println(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java index c82cafcb88..be6fef892a 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; -import org.apache.metron.common.configuration.SensorParserConfig; import org.json.simple.JSONObject; import org.junit.Before; import org.junit.Test; @@ -32,7 +31,6 @@ public class GrokWebSphereParserTest { private Map parserConfig; - private SensorParserConfig sensorParserConfig; @Before public void setup() { @@ -67,8 +65,6 @@ public void setup() { parserConfig.put("patternLabel", "WEBSPHERE"); parserConfig.put("timestampField", "timestamp_string"); parserConfig.put("dateFormat", "yyyy MMM dd HH:mm:ss"); - sensorParserConfig = new SensorParserConfig(); - sensorParserConfig.setParserConfig(parserConfig); } @Test @@ -79,7 +75,7 @@ public void testParseLoginLine() throws Exception { parser.configure(parserConfig); String testString = "<133>Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] user(rick007): " + "[120.43.200.6]: User logged into 'cohlOut'."; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -103,7 +99,7 @@ public void tetsParseLogoutLine() throws Exception { parser.configure(parserConfig); String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201]: " + "User 'hjpotter' logged out from 'default'."; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -126,7 +122,7 @@ public void tetsParseRBMLine() throws Exception { parser.configure(parserConfig); String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbm(RBM-Settings): " + "trans(3502888135)[request] gtid(3502888135): RBM: Resource access denied."; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -148,7 +144,7 @@ public void tetsParseOtherLine() throws Exception { parser.configure(parserConfig); String testString = "<134>Apr 15 17:17:34 SAGPXMLQA333 [0x8240001c][audit][info] trans(191): (admin:default:system:*): " + "ntp-service 'NTP Service' - Operational state down"; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -170,7 +166,7 @@ public void testParseMalformedLoginLine() throws Exception { parser.configure(parserConfig); String testString = "<133>Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] rick007): " + "[120.43.200. User logged into 'cohlOut'."; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -194,7 +190,7 @@ public void tetsParseMalformedLogoutLine() throws Exception { parser.configure(parserConfig); String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201: " + "User 'hjpotter' logged out from 'default."; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -217,7 +213,7 @@ public void tetsParseMalformedRBMLine() throws Exception { parser.configure(parserConfig); String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbmRBM-Settings): " + "trans3502888135)[request] gtid3502888135) RBM: Resource access denied."; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -239,7 +235,7 @@ public void tetsParseMalformedOtherLine() throws Exception { parser.configure(parserConfig); String testString = "<134>Apr 15 17:17:34 SAGPXMLQA333 [0x8240001c][audit][info] trans 191) admindefaultsystem*): " + "ntp-service 'NTP Service' - Operational state down:"; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -262,7 +258,7 @@ public void testParseEmptyLine() throws Exception { GrokWebSphereParser parser = new GrokWebSphereParser(); parser.configure(parserConfig); String testString = ""; - List result = parser.parse(testString.getBytes(), sensorParserConfig); + List result = parser.parse(testString.getBytes()); } } From 998a44c7a1ee6d5775cb4dfc42c3ab1ae478e61b Mon Sep 17 00:00:00 2001 From: rmerriman Date: Mon, 17 Oct 2016 15:05:13 -0500 Subject: [PATCH 6/7] fixed formatting to properly revert files to previous version --- .../sourcefire/BasicSourcefireParser.java | 2 +- .../metron/parsers/asa/GrokAsaParserTest.java | 22 +++++++-------- .../metron/parsers/bro/BroParserTest.java | 14 +++++----- .../fireeye/BasicFireEyeParserTest.java | 14 +++++----- .../parsers/ise/BasicIseParserTest.java | 18 ++++++------ .../lancope/BasicLancopeParserTest.java | 28 +++++++++---------- .../BasicPaloAltoFirewallParserTest.java | 14 +++++----- .../sourcefire/BasicSourcefireParserTest.java | 14 +++++----- 8 files changed, 63 insertions(+), 63 deletions(-) diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java index 28a298378c..0bc2671aa7 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java @@ -39,7 +39,7 @@ public class BasicSourcefireParser extends BasicParser { String domain_name_regex = "([^\\.]+)\\.([a-z]{2}|[a-z]{3}|([a-z]{2}\\.[a-z]{2}))$"; String sidRegex = "(.*)(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; //String sidRegex = "(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; - Pattern sidPattern = Pattern.compile(sidRegex); + Pattern sidPattern = Pattern.compile(sidRegex); Pattern pattern = Pattern.compile(domain_name_regex); @Override diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java index 39c895f689..8e9da0d65e 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/GrokAsaParserTest.java @@ -43,21 +43,21 @@ public class GrokAsaParserTest extends AbstractConfigTest{ * The grokAsaStrings. */ private static String[] grokAsaStrings=null; - + /** * The grokAsaParser. */ - + private GrokAsaParser grokAsaParser=null; - + /** * Constructs a new GrokAsaParserTest instance. * @throws Exception */ - + public GrokAsaParserTest() throws Exception { - super(); - + super(); + } /** * @throws java.lang.Exception @@ -72,19 +72,19 @@ public static void tearDownAfterClass() throws Exception { setGrokAsaStrings(null); } - /* + /* * (non-Javadoc) * @see junit.framework.TestCase#setUp() */ public void setUp() throws Exception { super.setUp("org.apache.metron.parsers.asa.GrokAsaParserTest"); setGrokAsaStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - grokAsaParser = new GrokAsaParser(); + grokAsaParser = new GrokAsaParser(); } /** - * - * + * + * * @throws java.lang.Exception */ public void tearDown() throws Exception { @@ -96,7 +96,7 @@ public void tearDown() throws Exception { */ @SuppressWarnings({ "rawtypes" }) public void testParse() { - + for (String grokAsaString : getGrokAsaStrings()) { JSONObject parsed = grokAsaParser.parse(grokAsaString.getBytes()).get(0); Assert.assertNotNull(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java index 9747f19120..2dd11c59bc 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BroParserTest.java @@ -50,8 +50,8 @@ * @version $Revision: 1.1 $ */ public class BroParserTest extends AbstractConfigTest { - - + + /** * The inputStrings. */ @@ -61,14 +61,14 @@ public class BroParserTest extends AbstractConfigTest { * The parser. */ private BasicBroParser parser=null; - + /** * Constructs a new BroParserTest instance. - * @throws Exception + * @throws Exception */ public BroParserTest() throws Exception { super(); - } + } /** @@ -89,9 +89,9 @@ public static void tearDownAfterClass() throws Exception { public void setUp() throws Exception { super.setUp("org.apache.metron.parsers.bro.BroParserTest"); setInputStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - parser = new BasicBroParser(); + parser = new BasicBroParser(); } - + /** * @throws ParseException * Tests for Parse Method diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java index a00dcd51f5..129619d6c9 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java @@ -43,17 +43,17 @@ public class BasicFireEyeParserTest extends AbstractConfigTest * The inputStrings. */ private static String[] inputStrings; - + /** * The parser. */ private BasicFireEyeParser parser=null; - + /** * Constructs a new BasicFireEyeParserTest instance. * @throws Exception - */ + */ public BasicFireEyeParserTest() throws Exception { super(); } @@ -77,17 +77,17 @@ public static void tearDownAfterClass() throws Exception { public void setUp() throws Exception { super.setUp("org.apache.metron.parsers.fireeye.BasicFireEyeParserTest"); setInputStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - parser = new BasicFireEyeParser(); + parser = new BasicFireEyeParser(); } /** - * - * + * + * * @throws java.lang.Exception */ public void tearDown() throws Exception { parser = null; - setInputStrings(null); + setInputStrings(null); } /** diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java index cac64e9ebf..751e4147f4 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java @@ -35,7 +35,7 @@ *
  • Description: Junit Test Case for BasicISE Parser
    • *
  • Created: AUG 25, 2014
    • * - * + * * @version $Revision: 1.1 $ */ @@ -43,7 +43,7 @@ public class BasicIseParserTest extends AbstractSchemaTest { /** * The inputStrings. */ - private static String[] inputStrings; + private static String[] inputStrings; /** * The parser. @@ -53,7 +53,7 @@ public class BasicIseParserTest extends AbstractSchemaTest { /** * Constructs a new BasicIseParserTest instance. - * + * * @param name */ @@ -62,14 +62,14 @@ public BasicIseParserTest(String name) { } /** - * + * * @throws java.lang.Exception */ protected static void setUpBeforeClass() throws Exception { } /** - * + * * @throws java.lang.Exception */ protected static void tearDownAfterClass() throws Exception { @@ -77,7 +77,7 @@ protected static void tearDownAfterClass() throws Exception { /* * (non-Javadoc) - * + * * @see junit.framework.TestCase#setUp() */ @@ -85,7 +85,7 @@ protected void setUp() throws Exception { super.setUp("org.apache.metron.parsers.lancope.BasicLancopeParserTest"); setInputStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); BasicIseParserTest.setIseParser(new BasicIseParser()); - + URL schema_url = getClass().getClassLoader().getResource( "TestSchemas/IseSchema.json"); super.setSchemaJsonString(super.readSchemaFromFile(schema_url)); @@ -93,7 +93,7 @@ protected void setUp() throws Exception { /* * (non-Javadoc) - * + * * @see junit.framework.TestCase#tearDown() */ @@ -104,7 +104,7 @@ protected void tearDown() throws Exception { /** * Test method for * {@link BasicIseParser#parse(byte[])}. - * + * * @throws IOException * @throws Exception */ diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java index 0d2bcb331d..4d8a2d0ff2 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java @@ -37,17 +37,17 @@ * @version $Revision: 1.1 $ */ public class BasicLancopeParserTest extends AbstractSchemaTest { - + /** * The inputStrings. */ - private static String[] inputStrings; + private static String[] inputStrings; /** * The parser. */ - private static BasicLancopeParser parser=null; + private static BasicLancopeParser parser=null; /** * Constructs a new BasicLancopeParserTest instance. @@ -59,20 +59,20 @@ public BasicLancopeParserTest(String name) { } /** - + * @throws java.lang.Exception */ - protected static void setUpBeforeClass() throws Exception { + protected static void setUpBeforeClass() throws Exception { } /** - + * @throws java.lang.Exception */ protected static void tearDownAfterClass() throws Exception { } - /* + /* * (non-Javadoc) * @see junit.framework.TestCase#setUp() */ @@ -80,14 +80,14 @@ protected static void tearDownAfterClass() throws Exception { protected void setUp() throws Exception { super.setUp("org.apache.metron.parsers.lancope.BasicLancopeParserTest"); setInputStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - BasicLancopeParserTest.setParser(new BasicLancopeParser()); - + BasicLancopeParserTest.setParser(new BasicLancopeParser()); + URL schema_url = getClass().getClassLoader().getResource( "TestSchemas/LancopeSchema.json"); - super.setSchemaJsonString(super.readSchemaFromFile(schema_url)); + super.setSchemaJsonString(super.readSchemaFromFile(schema_url)); } - /* + /* * (non-Javadoc) * @see junit.framework.TestCase#tearDown() */ @@ -98,11 +98,11 @@ protected void tearDown() throws Exception { /** * Test method for {@link BasicLancopeParser#parse(byte[])}. - * @throws Exception - * @throws IOException + * @throws Exception + * @throws IOException */ public void testParse() throws IOException, Exception { - + for (String inputString : getInputStrings()) { JSONObject parsed = parser.parse(inputString.getBytes()).get(0); assertNotNull(parsed); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java index 5dd27fcb1d..0c0947b2a6 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java @@ -37,18 +37,18 @@ public class BasicPaloAltoFirewallParserTest extends AbstractConfigTest { /** * Constructs a new BasicPaloAltoFirewallParserTest instance. * @throws Exception - */ + */ public BasicPaloAltoFirewallParserTest() throws Exception { - super(); + super(); } /** * Sets the inputStrings. * @param inputStrings the inputStrings. */ - + public static void setInputStrings(String[] inputStrings) { - + BasicPaloAltoFirewallParserTest.inputStrings = inputStrings; } @@ -76,12 +76,12 @@ public static void tearDownAfterClass() throws Exception { public void setUp() throws Exception { super.setUp("org.apache.metron.parsers.paloalto.BasicPaloAltoFirewallParserTest"); setPAStrings(super.readTestDataFromFile(this.getConfig().getString("logFile"))); - paParser = new BasicPaloAltoFirewallParser(); + paParser = new BasicPaloAltoFirewallParser(); } /** - * - * + * + * * @throws java.lang.Exception */ public void tearDown() throws Exception { diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java index 03a7f6e1bc..2ce035c238 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java @@ -41,9 +41,9 @@ public class BasicSourcefireParserTest extends AbstractConfigTest { /** * The sourceFireStrings. - */ + */ private static String[] sourceFireStrings; - + /** * The sourceFireParser. */ @@ -54,11 +54,11 @@ public class BasicSourcefireParserTest extends AbstractConfigTest * Constructs a new BasicSourcefireParserTest instance. * @throws Exception */ - + public BasicSourcefireParserTest() throws Exception { - super(); + super(); } - + /** * @throws java.lang.Exception */ @@ -82,8 +82,8 @@ public void setUp() throws Exception { } /** - * - * + * + * * @throws java.lang.Exception */ public void tearDown() throws Exception { From a673c21d733189914f818b808ebfe135887dcdf0 Mon Sep 17 00:00:00 2001 From: rmerriman Date: Tue, 18 Oct 2016 10:16:18 -0500 Subject: [PATCH 7/7] Put config update check and config read together at the beginning of the execute method. Also improved Zookeeper parser path checking on config update and added more test cases. --- .../org/apache/metron/parsers/bolt/ParserBolt.java | 13 +++++++++---- .../apache/metron/parsers/bolt/ParserBoltTest.java | 6 ++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 8df14dd224..d887b366f9 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -56,7 +56,7 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable { private MessageFilter filter = new GenericMessageFilter(); private WriterHandler writer; private org.apache.metron.common.dsl.Context stellarContext; - private AtomicBoolean configUpdatedFlag = new AtomicBoolean(false); + protected AtomicBoolean configUpdatedFlag = new AtomicBoolean(false); public ParserBolt( String zookeeperUrl , String sensorType , MessageParser parser @@ -115,15 +115,19 @@ protected void initializeStellar() { @Override public void execute(Tuple tuple) { byte[] originalMessage = tuple.getBinary(0); + + //Config update check and config read must be done together + boolean updateConfig = configUpdatedFlag.getAndSet(false); SensorParserConfig sensorParserConfig = getSensorParserConfig(); + try { //we want to ack the tuple in the situation where we have are not doing a bulk write //otherwise we want to defer to the writerComponent who will ack on bulk commit. boolean ackTuple = !writer.handleAck(); int numWritten = 0; if(sensorParserConfig != null) { - if (configUpdatedFlag.getAndSet(false)) { - parser.configurationUpdated(getSensorParserConfig().getParserConfig()); + if (updateConfig) { + parser.configurationUpdated(sensorParserConfig.getParserConfig()); } List fieldValidations = getConfigurations().getFieldValidations(); Optional> messages = parser.parseOptional(originalMessage); @@ -181,7 +185,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { @Override public void updateConfig(String path, byte[] data) throws IOException { super.updateConfig(path, data); - if (path.startsWith(ConfigurationType.PARSER.getZookeeperRoot() + "/" + getSensorType())) { + String pathWithoutTrailingSlash = path.replaceAll("/+$", ""); + if (pathWithoutTrailingSlash.equals(ConfigurationType.PARSER.getZookeeperRoot() + "/" + getSensorType())) { configUpdatedFlag.set(true); } } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index 1dcd238cf9..1f8863c13d 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -51,6 +51,8 @@ import java.util.Optional; import static org.junit.Assert.fail; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; @@ -262,6 +264,10 @@ protected ParserConfigurations defaultConfigurations() { verify(parser, times(0)).configurationUpdated(any()); verify(outputCollector, times(1)).emit(eq(Constants.INVALID_STREAM), eq(new Values(finalMessage2))); + parserBolt.updateConfig(ConfigurationType.PARSER.getZookeeperRoot() + "/" + sensorType + "test", Bytes.toBytes(updatedParserConfig)); + assertFalse("Update flag should not be set when sensor name is a substring of another sensor", parserBolt.configUpdatedFlag.get()); + parserBolt.updateConfig(ConfigurationType.PARSER.getZookeeperRoot() + "/" + sensorType + "/", Bytes.toBytes(updatedParserConfig)); + assertTrue("Update flag should be set even if path has a trailing slash", parserBolt.configUpdatedFlag.get()); parserBolt.updateConfig(ConfigurationType.PARSER.getZookeeperRoot() + "/" + sensorType, Bytes.toBytes(updatedParserConfig)); parserBolt.execute(tuple); verify(writer, times(2)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1));