diff --git a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml b/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml deleted file mode 100644 index d857bf5d2c..0000000000 --- a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create HDFS directory for grok patterns - command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Assign hfds user as owner of {{ metron_hdfs_output_dir }}/patterns HDFS directory - command: hdfs dfs -chown -R hdfs:hadoop {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Assign permissions of HDFS {{ metron_hdfs_output_dir }}/patterns directory - command: hdfs dfs -chmod -R 775 {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Upload Grok Patterns to hdfs://{{ metron_hdfs_output_dir }} - command: hdfs dfs -put -f {{ metron_directory }}/patterns {{ metron_hdfs_output_dir }} - become: yes - become_user: hdfs - diff --git a/metron-deployment/roles/metron_streaming/tasks/main.yml b/metron-deployment/roles/metron_streaming/tasks/main.yml index 0945e3d6b8..63e092fc77 100644 --- a/metron-deployment/roles/metron_streaming/tasks/main.yml +++ b/metron-deployment/roles/metron_streaming/tasks/main.yml @@ -33,9 +33,6 @@ - include: hdfs_filesystem.yml run_once: true -- include: grok_upload.yml - run_once: true - - include: topologies.yml - include: source_config.yml diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json index e44c4c28ea..beec36cb16 100644 --- a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json +++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json @@ -2,7 +2,7 @@ "parserClassName": "org.apache.metron.parsers.GrokParser", "sensorTopic": "squid", "parserConfig": { - "grokPath": "/patterns/squid", + "grokPattern": "SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?", "patternLabel": "SQUID_DELIMITED", "timestampField": "timestamp" }, diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json index 0f2c901415..9d9b111934 100644 --- a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json +++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json @@ -1,11 +1,35 @@ { - "parserClassName":"org.apache.metron.parsers.websphere.GrokWebSphereParser", - "sensorTopic":"websphere", - "parserConfig": - { - "grokPath":"/patterns/websphere", - "patternLabel":"WEBSPHERE", - "timestampField":"timestamp_string", - "dateFormat":"yyyy MMM dd HH:mm:ss" + "parserClassName": "org.apache.metron.parsers.websphere.GrokWebSphereParser", + "sensorTopic": "websphere", + "parserConfig": { + "grokPattern": [ + "# Days - two digit number is used", + "DAY \\d{1,2}", + "# Time - two digit hour, minute, and second", + "TIME \\d{2}:\\d{2}:\\d{2}", + "# Timestamp - month, day, and time", + "TIMESTAMP %{MONTH:UNWANTED}\\s+%{DAY:UNWANTED} %{TIME:UNWANTED}", + "# Generic word field", + "WORD \\w+", + "# Priority", + "PRIORITY \\d+", + "# Log start - the first part of the log line", + "LOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname}", + "# Security domain", + "SECURITY_DOMAIN [%{WORD:security_domain}]", + "# Log middle - the middle part of the log line", + "LOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]", + "# Define IP address formats", + "IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?", + "IPV4 (?, Serializable { protected static final Logger LOG = LoggerFactory.getLogger(GrokParser.class); protected transient Grok grok; - protected String grokPath; + protected String grokPattern; protected String patternLabel; protected List timeFields = new ArrayList<>(); protected String timestampField; protected SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z"); - protected String patternsCommonDir = "/patterns/common"; + protected String patternsCommonPath = "/patterns/common"; + @SuppressWarnings("unchecked") @Override public void configure(Map parserConfig) { - this.grokPath = (String) parserConfig.get("grokPath"); + Object grokPattern = parserConfig.get("grokPattern"); + if (grokPattern instanceof String) { + this.grokPattern = (String) grokPattern; + } else if (grokPattern instanceof String[]){ + String[] patterns = (String[]) grokPattern; + this.grokPattern = Joiner.on('\n').join(patterns); + } else if (grokPattern instanceof Iterable) { + Iterable patterns = (Iterable) grokPattern; + this.grokPattern = Joiner.on('\n').join(patterns); + } this.patternLabel = (String) parserConfig.get("patternLabel"); this.timestampField = (String) parserConfig.get("timestampField"); List timeFieldsParam = (List) parserConfig.get("timeFields"); @@ -77,41 +84,30 @@ public void configure(Map parserConfig) { } } - public InputStream openInputStream(String streamName) throws IOException { - FileSystem fs = FileSystem.get(new Configuration()); - Path path = new Path(streamName); - if(fs.exists(path)) { - return fs.open(path); - } else { - return getClass().getResourceAsStream(streamName); - } - } - @Override public void init() { grok = new Grok(); try { - InputStream commonInputStream = openInputStream(patternsCommonDir); + InputStream commonInputStream = getClass().getResourceAsStream(patternsCommonPath); if (LOG.isDebugEnabled()) { - LOG.debug("Grok parser loading common patterns from: " + patternsCommonDir); + LOG.debug("Grok parser loading common patterns from: " + patternsCommonPath); } if (commonInputStream == null) { throw new RuntimeException( - "Unable to initialize grok parser: Unable to load " + patternsCommonDir + " from either classpath or HDFS"); + "Unable to initialize grok parser: Unable to load " + patternsCommonPath + " from classpath"); } grok.addPatternFromReader(new InputStreamReader(commonInputStream)); + if (LOG.isDebugEnabled()) { - LOG.debug("Loading parser-specific patterns from: " + grokPath); + LOG.debug("Loading parser-specific patterns: " + grokPattern); } - InputStream patterInputStream = openInputStream(grokPath); - if (patterInputStream == null) { - throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + grokPath - + " from either classpath or HDFS"); + if (grokPattern == null) { + throw new RuntimeException("Unable to initialize grok parser: grokPattern config property is empty"); } - grok.addPatternFromReader(new InputStreamReader(patterInputStream)); + grok.addPatternFromReader(new InputStreamReader(new ByteArrayInputStream(grokPattern.getBytes()))); if (LOG.isDebugEnabled()) { LOG.debug("Grok parser set the following grok expression: " + grok.getNamedRegexCollectionById(patternLabel)); @@ -150,8 +146,8 @@ public List parse(byte[] rawMessage) { if (message.size() == 0) throw new RuntimeException("Grok statement produced a null message. Original message was: " - + originalMessage + " and the parsed message was: " + message + " . Check the pattern at: " - + grokPath); + + originalMessage + " , parsed message was: " + message + " , pattern was: " + + (LOG.isDebugEnabled() ? grokPattern : (patternLabel + " (Turn on DEBUG logging to see pattern text.)"))); message.put("original_string", originalMessage); for (String timeField : timeFields) { @@ -199,6 +195,12 @@ public boolean validate(JSONObject message) { return false; } + @Override + public void configurationUpdated(Map parserConfig) { + configure(parserConfig); + init(); + } + protected void postParse(JSONObject message) {} protected long toEpoch(String datetime) throws ParseException { diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 325209f880..d887b366f9 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -25,23 +25,28 @@ import backtype.storm.tuple.Values; import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredParserBolt; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.FieldValidator; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.FunctionResolver; import org.apache.metron.common.dsl.StellarFunctions; +import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.parsers.filters.Filters; -import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.parsers.filters.GenericMessageFilter; -import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.parsers.interfaces.MessageFilter; import org.apache.metron.parsers.interfaces.MessageParser; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; public class ParserBolt extends ConfiguredParserBolt implements Serializable { @@ -51,6 +56,7 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable { private MessageFilter filter = new GenericMessageFilter(); private WriterHandler writer; private org.apache.metron.common.dsl.Context stellarContext; + protected AtomicBoolean configUpdatedFlag = new AtomicBoolean(false); public ParserBolt( String zookeeperUrl , String sensorType , MessageParser parser @@ -109,13 +115,20 @@ protected void initializeStellar() { @Override public void execute(Tuple tuple) { byte[] originalMessage = tuple.getBinary(0); + + //Config update check and config read must be done together + boolean updateConfig = configUpdatedFlag.getAndSet(false); SensorParserConfig sensorParserConfig = getSensorParserConfig(); + try { //we want to ack the tuple in the situation where we have are not doing a bulk write //otherwise we want to defer to the writerComponent who will ack on bulk commit. boolean ackTuple = !writer.handleAck(); int numWritten = 0; if(sensorParserConfig != null) { + if (updateConfig) { + parser.configurationUpdated(sensorParserConfig.getParserConfig()); + } List fieldValidations = getConfigurations().getFieldValidations(); Optional> messages = parser.parseOptional(originalMessage); for (JSONObject message : messages.orElse(Collections.emptyList())) { @@ -168,4 +181,13 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declareStream(Constants.INVALID_STREAM, new Fields("message")); declarer.declareStream(Constants.ERROR_STREAM, new Fields("message")); } + + @Override + public void updateConfig(String path, byte[] data) throws IOException { + super.updateConfig(path, data); + String pathWithoutTrailingSlash = path.replaceAll("/+$", ""); + if (pathWithoutTrailingSlash.equals(ConfigurationType.PARSER.getZookeeperRoot() + "/" + getSensorType())) { + configUpdatedFlag.set(true); + } + } } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java index e3b903ee07..9157b75cc2 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java @@ -17,7 +17,6 @@ */ package org.apache.metron.parsers.interfaces; -import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Optional; @@ -52,4 +51,7 @@ default Optional> parseOptional(byte[] parseMessage) { */ boolean validate(T message); + default void configurationUpdated(Map config) { + } + } diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/asa b/metron-platform/metron-parsers/src/main/resources/patterns/asa deleted file mode 100644 index 8c2da93e6b..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/asa +++ /dev/null @@ -1,176 +0,0 @@ -# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns - -USERNAME [a-zA-Z0-9._-]+ -USER %{USERNAME:UNWANTED} -INT (?:[+-]?(?:[0-9]+)) -BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) -NUMBER (?:%{BASE10NUM:UNWANTED}) -BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) -UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} - -# Networking -MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) -CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) -WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) -COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (?/(?>[\w_%!$@:.,~-]+|\\.)*)+ -#UNIXPATH (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ -URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? -URIHOST %{IPORHOST}(?::%{POSINT:port})? -# uripath comes loosely from RFC1738, but mostly from what Firefox -# doesn't turn into %XX -URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ -#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? -URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* -URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? -URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? - -# Months: January, Feb, 3, 03, 12, December -MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b -MONTHNUM (?:0?[1-9]|1[0-2]) -MONTHNUM2 (?:0[1-9]|1[0-2]) -MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) - -# Days: Monday, Tue, Thu, etc... -DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) - -# Years? -YEAR (?>\d\d){1,2} -# Time: HH:MM:SS -#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? -# I'm still on the fence about using grok to perform the time match, -# since it's probably slower. -# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? -HOUR (?:2[0123]|[01]?[0-9]) -MINUTE (?:[0-5][0-9]) -# '60' is a leap second in most time standards and thus is valid. -SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) -TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) -# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) -DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} -DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} -ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) -ISO8601_SECOND (?:%{SECOND}|60) -TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? -DATE %{DATE_US}|%{DATE_EU} -DATESTAMP %{DATE}[- ]%{TIME} -TZ (?:[PMCE][SD]T|UTC) -DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} -DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} -DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} -DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} -GREEDYDATA .* - -# Syslog Dates: Month Day HH:MM:SS -SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} -PROG (?:[\w._/%-]+) -SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? -SYSLOGHOST %{IPORHOST} -SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> -HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} - -# Shortcuts -QS %{QUOTEDSTRING:UNWANTED} - -# Log formats -SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: - -MESSAGESLOG %{SYSLOGBASE} %{DATA} - -COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) -COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} - -# Log Levels -LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) - -#== Cisco ASA == -CISCO_TAGGED_SYSLOG ^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:sysloghost})? ?:? %%{CISCOTAG:ciscotag}: -CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} -CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) - -# Common Particles -CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted -CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* -CISCO_DIRECTION Inbound|inbound|Outbound|outbound -CISCO_INTERVAL first hit|%{INT}-second interval -CISCO_XLATE_TYPE static|dynamic -# ASA-2-106001 -CISCOFW106001 : %{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} -# ASA-2-106006, ASA-2-106007, ASA-2-106010 -CISCOFW106006_106007_106010 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}(\(%{DATA:dst_fwuser}\))? (?:on interface %{DATA:interface}|due to %{CISCO_REASON:reason}) -# ASA-3-106014 -CISCOFW106014 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\) -# ASA-6-106015 -CISCOFW106015 : %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} -# ASA-1-106021 -CISCOFW106021 : %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} -# ASA-4-106023 -CISCOFW106023 : %{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group %{DATA:policy_id} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -# ASA-5-106100 -CISCOFW106100 : access-list %{WORD:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -# ASA-6-110002 -CISCOFW110002 : %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} -# ASA-6-302010 -CISCOFW302010 : %{INT:connection_count} in use, %{INT:connection_count_max} most used -# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 -CISCOFW302013_302014_302015_302016 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))? -# ASA-6-302020, ASA-6-302021 -CISCOFW302020_302021 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))? -# ASA-6-305011 -CISCOFW305011 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} -# ASA-3-313001, ASA-3-313004, ASA-3-313008 -CISCOFW313001_313004_313008 : %{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? -# ASA-4-313005 -CISCOFW313005 : %{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type}, code %{INT:err_icmp_code}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}(\(%{DATA:orig_dst_fwuser}\))? -# ASA-4-402117 -CISCOFW402117 : %{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip} -# ASA-4-402119 -CISCOFW402119 : %{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking -# ASA-4-419001 -CISCOFW419001 : %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason} -# ASA-4-419002 -CISCOFW419002 : %{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number -# ASA-4-500004 -CISCOFW500004 : %{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} -# ASA-6-602303, ASA-6-602304 -CISCOFW602303_602304 : %{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action} -# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 -CISCOFW710001_710002_710003_710005_710006 : %{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} -# ASA-6-713172 -CISCOFW713172 : Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device -# ASA-4-733100 -CISCOFW733100 : \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count} - - -# ASA-6-305012 -CISCOFW305012 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} duration %{TIME:duration} -# ASA-7-609001 -CISCOFW609001 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? -# ASA-7-609002 -CISCOFW609002 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? duration %{TIME:duration} - - -#== End Cisco ASA == \ No newline at end of file diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/fireeye b/metron-platform/metron-parsers/src/main/resources/patterns/fireeye deleted file mode 100644 index 5dc99bfa4f..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/fireeye +++ /dev/null @@ -1,9 +0,0 @@ -GREEDYDATA .* -POSINT \b(?:[1-9][0-9]*)\b -UID [0-9.]+ -DATA .*? - -FIREEYE_BASE ^<%{POSINT:syslog_pri}>fenotify-%{UID:uid}.alert: %{GREEDYDATA:syslog} -FIREEYE_MAIN <%{POSINT:syslog_pri}>fenotify-%{DATA:uid}.alert: %{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{GREEDYDATA:fedata} -#\|(.?)\|(.?)\|(.?)\|(.?)\|%{DATA:type}\|(.?)\|%{GREEDYDATA:fedata} -FIREEYE_SUB ^<%{POSINT:syslog_pri}>fenotify-%{UID:uid}.alert: .?*\|.?*\|.?*\|.?*\|.?*\|%{DATA:type}\|.?*\|%{GREEDYDATA:fedata} diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire b/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire deleted file mode 100644 index 672f68435e..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire +++ /dev/null @@ -1,30 +0,0 @@ -POSINT \b(?:[1-9][0-9]*)\b -NONNEGINT \b(?:[0-9]+)\b -WORD \b\w+\b -NOTSPACE \S+ -SPACE \s* -DATA .*? -GREEDYDATA .* -QUOTEDSTRING (?>(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) -UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} - -# Networking -MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) -CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) -WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) -COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (?\s%{ip_dst_addr}\:%{ip_dst_port} \ No newline at end of file diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/squid b/metron-platform/metron-parsers/src/main/resources/patterns/squid deleted file mode 100644 index bf6c5b7ee9..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/squid +++ /dev/null @@ -1,2 +0,0 @@ -SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})? - diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/websphere b/metron-platform/metron-parsers/src/main/resources/patterns/websphere deleted file mode 100644 index 546944c1c5..0000000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/websphere +++ /dev/null @@ -1,37 +0,0 @@ -# Months - only three-letter code is used -MONTH \b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec?)\b - -# Days - two digit number is used -DAY \d{1,2} - -# Time - two digit hour, minute, and second -TIME \d{2}:\d{2}:\d{2} - -# Timestamp - month, day, and time -TIMESTAMP %{MONTH:UNWANTED}\s+%{DAY:UNWANTED} %{TIME:UNWANTED} - -# Generic word field -WORD \w+ - -# Priority -PRIORITY \d+ - -# Log start - the first part of the log line -LOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname} - -# Security domain -SECURITY_DOMAIN [%{WORD:security_domain}] - -# Log middle - the middle part of the log line -LOGMIDDLE (\[%{WORD:security_domain}\])?\[%{WORD:event_code}\]\[%{WORD:event_type}\]\[%{WORD:severity}\] - -# Define IP address formats -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (? parserConfig = new HashMap<>(); - parserConfig.put("grokPath", getGrokPath()); + parserConfig.put("grokPattern", getGrokPattern()); parserConfig.put("patternLabel", getGrokPatternLabel()); parserConfig.put("timestampField", getTimestampField()); parserConfig.put("dateFormat", getDateFormat()); @@ -87,7 +87,7 @@ public boolean compare(JSONObject expected, JSONObject actual) { } public abstract Map getTestData(); - public abstract String getGrokPath(); + public abstract Object getGrokPattern(); public abstract String getGrokPatternLabel(); public abstract List getTimeFields(); public abstract String getDateFormat(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java index 89eb30c24f..90f2d4b311 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java @@ -17,7 +17,11 @@ */ package org.apache.metron.parsers; +import junit.framework.Assert; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.json.simple.JSONObject; +import org.junit.Test; import java.util.ArrayList; import java.util.HashMap; @@ -68,8 +72,10 @@ public Map getTestData() { } - public String getGrokPath() { - return "../metron-integration-test/src/main/sample/patterns/test"; + public String[] getGrokPattern() { + String[] grokPattern = {"YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}", + "YAF_DELIMITED %{NUMBER:start_time}\\|%{YAF_TIME_FORMAT:end_time}\\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\\|%{SPACE:UNWANTED}%{INT:protocol}\\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\\|%{SPACE:UNWANTED}%{INT:ip_src_port}\\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\\|%{SPACE:UNWANTED}%{DATA:iflags}\\|%{SPACE:UNWANTED}%{DATA:uflags}\\|%{SPACE:UNWANTED}%{DATA:riflags}\\|%{SPACE:UNWANTED}%{DATA:ruflags}\\|%{SPACE:UNWANTED}%{WORD:isn}\\|%{SPACE:UNWANTED}%{DATA:risn}\\|%{SPACE:UNWANTED}%{DATA:tag}\\|%{GREEDYDATA:rtag}\\|%{SPACE:UNWANTED}%{INT:pkt}\\|%{SPACE:UNWANTED}%{INT:oct}\\|%{SPACE:UNWANTED}%{INT:rpkt}\\|%{SPACE:UNWANTED}%{INT:roct}\\|%{SPACE:UNWANTED}%{INT:app}\\|%{GREEDYDATA:end_reason}"}; + return grokPattern; } public String getGrokPatternLabel() { @@ -89,4 +95,37 @@ public String getDateFormat() { public String getTimestampField() { return "start_time"; } + + @Test + public void testConfigChange() { + String raw = "123 test"; + String pattern1 = "LABEL %{NUMBER:field_1}"; + String pattern2 = "LABEL %{NUMBER:field_1} %{WORD:field_2}"; + JSONObject expected1 = new JSONObject(); + expected1.put("field_1", 123); + expected1.put("original_string", raw); + JSONObject expected2 = new JSONObject(); + expected2.put("field_1", 123); + expected2.put("field_2", "test"); + expected2.put("original_string", raw); + Map parserConfig = new HashMap<>(); + parserConfig.put("grokPattern", pattern1); + parserConfig.put("patternLabel", "LABEL"); + SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setParserConfig(parserConfig); + + GrokParser grokParser = new GrokParser(); + grokParser.configure(parserConfig); + grokParser.init(); + + List results = grokParser.parse(raw.getBytes()); + Assert.assertEquals(1, results.size()); + compare(expected1, results.get(0)); + + parserConfig.put("grokPattern", pattern2); + grokParser.configurationUpdated(sensorParserConfig.getParserConfig()); + results = grokParser.parse(raw.getBytes()); + Assert.assertEquals(1, results.size()); + compare(expected2, results.get(0)); + } } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java index 93c8276a2b..846e55f402 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java @@ -75,8 +75,8 @@ public Map getTestData() { @Override - public String getGrokPath() { - return "../metron-parsers/src/main/resources/patterns/squid"; + public String getGrokPattern() { + return "SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?"; } @Override diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java index 8dd75a02d5..4c9af2ae34 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java @@ -69,8 +69,10 @@ public Map getTestData() { } @Override - public String getGrokPath() { - return "../metron-parsers/src/main/resources/patterns/yaf"; + public String[] getGrokPattern() { + String[] grokPattern = {"YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}", + "YAF_DELIMITED %{YAF_TIME_FORMAT:start_time}\\|%{YAF_TIME_FORMAT:end_time}\\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\\|%{SPACE:UNWANTED}%{INT:protocol}\\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\\|%{SPACE:UNWANTED}%{INT:ip_src_port}\\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\\|%{SPACE:UNWANTED}%{DATA:iflags}\\|%{SPACE:UNWANTED}%{DATA:uflags}\\|%{SPACE:UNWANTED}%{DATA:riflags}\\|%{SPACE:UNWANTED}%{DATA:ruflags}\\|%{SPACE:UNWANTED}%{WORD:isn}\\|%{SPACE:UNWANTED}%{DATA:risn}\\|%{SPACE:UNWANTED}%{DATA:tag}\\|%{GREEDYDATA:rtag}\\|%{SPACE:UNWANTED}%{INT:pkt}\\|%{SPACE:UNWANTED}%{INT:oct}\\|%{SPACE:UNWANTED}%{INT:rpkt}\\|%{SPACE:UNWANTED}%{INT:roct}\\|%{SPACE:UNWANTED}%{INT:app}\\|%{GREEDYDATA:end_reason}"}; + return grokPattern; } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index 5d938384fc..1f8863c13d 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -17,28 +17,26 @@ */ package org.apache.metron.parsers.bolt; -import org.apache.metron.common.configuration.SensorParserConfig; - import backtype.storm.task.OutputCollector; import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; import com.google.common.collect.ImmutableList; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ParserConfigurations; +import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.dsl.Context; import org.apache.metron.common.writer.BulkMessageWriter; -import org.adrianwalker.multilinestring.Multiline; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.metron.common.configuration.ParserConfigurations; -import org.apache.metron.common.configuration.SensorParserConfig; -import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.common.writer.BulkWriterResponse; +import org.apache.metron.common.writer.MessageWriter; import org.apache.metron.parsers.BasicParser; -import org.apache.metron.parsers.csv.CSVParser; -import org.apache.metron.test.bolt.BaseBoltTest; import org.apache.metron.parsers.interfaces.MessageFilter; import org.apache.metron.parsers.interfaces.MessageParser; -import org.apache.metron.common.writer.MessageWriter; +import org.apache.metron.test.bolt.BaseBoltTest; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.junit.Assert; @@ -46,8 +44,15 @@ import org.mockito.Mock; import java.io.IOException; -import java.util.*; - +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.fail; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; @@ -147,33 +152,98 @@ public Map getParserConfig() { verify(outputCollector, times(1)).ack(tuple); } + /** + { + "fieldValidations" : [ + { + "input" : [ "field1" ], + "validation" : "NOT_EMPTY" + } + ] + } + */ + @Multiline + public static String globalConfig; + + + /** + { + "sensorTopic": "yaf", + "filterClassName": "org.apache.metron.parsers.filters.GenericMessageFilter", + "parserConfig" : { + "config" : "value" + }, + "fieldTransformations" : [ + { + "transformation" : "STELLAR" + ,"output" : [ "field3" ] + ,"config" : { + "field3" : "TO_UPPER(field1)" + } + } + ] + } + */ + @Multiline + public static String sensorParserConfig; + + /** + { + "sensorTopic": "yaf", + "parserConfig" : { + "config" : "updatedValue" + } + } + */ + @Multiline + public static String updatedParserConfig; + + @Test + public void testPrepare() throws Exception { + String sensorType = "yaf"; + ParserConfigurations parserConfigurations = new ParserConfigurations(); + ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) { + @Override + protected ParserConfigurations defaultConfigurations() { + return parserConfigurations; + } + }; + parserBolt.setCuratorFramework(client); + parserBolt.setTreeCache(cache); + try { + parserBolt.prepare(new HashMap(), topologyContext, outputCollector); + fail("ParserBolt.prepare should throw exception on empty config"); + } catch(IllegalStateException e) { + } + + parserConfigurations.updateSensorParserConfig(sensorType, Bytes.toBytes(sensorParserConfig)); + parserBolt.withMessageFilter(null); + parserBolt.prepare(new HashMap(), topologyContext, outputCollector); + verify(parser, times(2)).init(); + verify(writer, times(2)).init(); + } + @Test public void test() throws Exception { String sensorType = "yaf"; + + ParserConfigurations parserConfigurations = new ParserConfigurations(); + parserConfigurations.updateGlobalConfig(Bytes.toBytes(globalConfig)); + parserConfigurations.updateSensorParserConfig(sensorType, Bytes.toBytes(sensorParserConfig)); ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) { @Override protected ParserConfigurations defaultConfigurations() { - return new ParserConfigurations() { - @Override - public SensorParserConfig getSensorParserConfig(String sensorType) { - return new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - }}; - } - }; - } - }; + return parserConfigurations; } }; parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); - verify(parser, times(1)).init(); - verify(writer, times(1)).init(); + parserBolt.declareOutputFields(declarer); + verify(declarer, times(1)).declareStream(eq(Constants.INVALID_STREAM), any()); + verify(declarer, times(1)).declareStream(eq(Constants.ERROR_STREAM), any()); byte[] sampleBinary = "some binary message".getBytes(); JSONParser jsonParser = new JSONParser(); final JSONObject sampleMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\" }"); @@ -182,61 +252,74 @@ public Map getParserConfig() { add(sampleMessage1); add(sampleMessage2); }}; - final JSONObject finalMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"source.type\":\"" + sensorType + "\" }"); - final JSONObject finalMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"source.type\":\"" + sensorType + "\" }"); + final JSONObject finalMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"field3\":\"VALUE1\", \"source.type\":\"" + sensorType + "\" }"); + final JSONObject finalMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"source.type\":\"" + sensorType + ".invalid\" }"); when(tuple.getBinary(0)).thenReturn(sampleBinary); when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messages)); when(parser.validate(eq(messages.get(0)))).thenReturn(true); - when(parser.validate(eq(messages.get(1)))).thenReturn(false); + when(parser.validate(eq(messages.get(1)))).thenReturn(true); parserBolt.execute(tuple); verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1)); verify(outputCollector, times(1)).ack(tuple); - when(parser.validate(eq(messages.get(0)))).thenReturn(true); - when(parser.validate(eq(messages.get(1)))).thenReturn(true); - when(filter.emitTuple(eq(messages.get(0)), any())).thenReturn(false); - when(filter.emitTuple(eq(messages.get(1)), any())).thenReturn(true); - parserBolt.withMessageFilter(filter); + verify(parser, times(0)).configurationUpdated(any()); + verify(outputCollector, times(1)).emit(eq(Constants.INVALID_STREAM), eq(new Values(finalMessage2))); + + parserBolt.updateConfig(ConfigurationType.PARSER.getZookeeperRoot() + "/" + sensorType + "test", Bytes.toBytes(updatedParserConfig)); + assertFalse("Update flag should not be set when sensor name is a substring of another sensor", parserBolt.configUpdatedFlag.get()); + parserBolt.updateConfig(ConfigurationType.PARSER.getZookeeperRoot() + "/" + sensorType + "/", Bytes.toBytes(updatedParserConfig)); + assertTrue("Update flag should be set even if path has a trailing slash", parserBolt.configUpdatedFlag.get()); + parserBolt.updateConfig(ConfigurationType.PARSER.getZookeeperRoot() + "/" + sensorType, Bytes.toBytes(updatedParserConfig)); parserBolt.execute(tuple); - verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2)); + verify(writer, times(2)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1)); verify(outputCollector, times(2)).ack(tuple); - doThrow(new Exception()).when(writer).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2)); + verify(parser, times(1)).configurationUpdated(SensorParserConfig.fromBytes(Bytes.toBytes(updatedParserConfig)).getParserConfig()); + + when(filter.emitTuple(eq(messages.get(0)), any())).thenReturn(true); + when(filter.emitTuple(eq(messages.get(1)), any())).thenReturn(false); + parserBolt.withMessageFilter(filter); + parserBolt.execute(tuple); + verify(writer, times(3)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1)); + verify(outputCollector, times(3)).ack(tuple); + doThrow(new Exception()).when(writer).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1)); parserBolt.execute(tuple); verify(outputCollector, times(1)).reportError(any(Throwable.class)); } -@Test -public void testImplicitBatchOfOne() throws Exception { - String sensorType = "yaf"; + @Test + public void testImplicitBatchOfOne() throws Exception { - ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { - @Override - protected ParserConfigurations defaultConfigurations() { - return new ParserConfigurations() { - @Override - public SensorParserConfig getSensorParserConfig(String sensorType) { - return new SensorParserConfig() { - @Override - public Map getParserConfig() { - return new HashMap() {{ - }}; - } - }; - } - }; - } - }; - parserBolt.setCuratorFramework(client); - parserBolt.setTreeCache(cache); - parserBolt.prepare(new HashMap(), topologyContext, outputCollector); - verify(parser, times(1)).init(); - verify(batchWriter, times(1)).init(any(), any()); - when(parser.validate(any())).thenReturn(true); - when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); - when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); - parserBolt.withMessageFilter(filter); - parserBolt.execute(t1); - verify(outputCollector, times(1)).ack(t1); -} + String sensorType = "yaf"; + + ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) { + @Override + protected ParserConfigurations defaultConfigurations() { + return new ParserConfigurations() { + @Override + public SensorParserConfig getSensorParserConfig(String sensorType) { + return new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + }}; + } + }; + } + }; + } + }; + parserBolt.setCuratorFramework(client); + parserBolt.setTreeCache(cache); + parserBolt.prepare(new HashMap(), topologyContext, outputCollector); + verify(parser, times(1)).init(); + verify(batchWriter, times(1)).init(any(), any()); + when(parser.validate(any())).thenReturn(true); + when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject()))); + when(filter.emitTuple(any(), any(Context.class))).thenReturn(true); + parserBolt.withMessageFilter(filter); + parserBolt.execute(t1); + verify(outputCollector, times(1)).ack(t1); + verify(parser, times(0)).configurationUpdated(any()); + } /** { @@ -247,7 +330,7 @@ public Map getParserConfig() { } */ @Multiline - public static String sensorParserConfig; + public static String filterSensorParserConfig; @Test public void testFilter() throws Exception { String sensorType = "yaf"; @@ -256,7 +339,7 @@ public void testFilter() throws Exception { @Override protected SensorParserConfig getSensorParserConfig() { try { - return SensorParserConfig.fromBytes(Bytes.toBytes(sensorParserConfig)); + return SensorParserConfig.fromBytes(Bytes.toBytes(filterSensorParserConfig)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java index 87afe10002..be6fef892a 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java @@ -34,8 +34,34 @@ public class GrokWebSphereParserTest { @Before public void setup() { + String[] grokPattern = {"# Months - only three-letter code is used", + "MONTH \\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec?)\\b", + "# Days - two digit number is used", + "DAY \\d{1,2}", + "# Time - two digit hour, minute, and second", + "TIME \\d{2}:\\d{2}:\\d{2}", + "# Timestamp - month, day, and time", + "TIMESTAMP %{MONTH:UNWANTED}\\s+%{DAY:UNWANTED} %{TIME:UNWANTED}", + "# Generic word field", + "WORD \\w+", + "# Priority", + "PRIORITY \\d+", + "# Log start - the first part of the log line", + "LOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname}", + "# Security domain", + "SECURITY_DOMAIN [%{WORD:security_domain}]", + "# Log middle - the middle part of the log line", + "LOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]", + "# Define IP address formats", + "IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?", + "IPV4 (?(); - parserConfig.put("grokPath", "../metron-parsers/src/main/resources/patterns/websphere"); + parserConfig.put("grokPattern", grokPattern); parserConfig.put("patternLabel", "WEBSPHERE"); parserConfig.put("timestampField", "timestamp_string"); parserConfig.put("dateFormat", "yyyy MMM dd HH:mm:ss"); @@ -140,7 +166,7 @@ public void testParseMalformedLoginLine() throws Exception { parser.configure(parserConfig); String testString = "<133>Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] rick007): " + "[120.43.200. User logged into 'cohlOut'."; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes()); JSONObject parsedJSON = result.get(0); //Compare fields @@ -232,7 +258,7 @@ public void testParseEmptyLine() throws Exception { GrokWebSphereParser parser = new GrokWebSphereParser(); parser.configure(parserConfig); String testString = ""; - List result = parser.parse(testString.getBytes()); + List result = parser.parse(testString.getBytes()); } }