diff --git a/google_checks.xml b/google_checks.xml
index 5aa8996..ad4fc71 100644
--- a/google_checks.xml
+++ b/google_checks.xml
@@ -67,6 +67,7 @@
+
diff --git a/recog-verify/src/main/java/com/rapid7/recog/verify/RecogVerifier.java b/recog-verify/src/main/java/com/rapid7/recog/verify/RecogVerifier.java
index c6d1eee..1567bef 100644
--- a/recog-verify/src/main/java/com/rapid7/recog/verify/RecogVerifier.java
+++ b/recog-verify/src/main/java/com/rapid7/recog/verify/RecogVerifier.java
@@ -59,10 +59,10 @@ public void verify() {
matcher.verifyExamples((status, message) -> {
switch (status) {
case Warn:
- reporter.warning(String.format("WARN: %s", message));
+ reporter.warning(message, matcher.getLine());
break;
case Fail:
- reporter.failure(String.format("FAIL: %s", message));
+ reporter.failure(message, matcher.getLine());
break;
case Success:
reporter.success(message);
diff --git a/recog-verify/src/main/java/com/rapid7/recog/verify/VerifyReporter.java b/recog-verify/src/main/java/com/rapid7/recog/verify/VerifyReporter.java
index 4ba8572..307e2a0 100644
--- a/recog-verify/src/main/java/com/rapid7/recog/verify/VerifyReporter.java
+++ b/recog-verify/src/main/java/com/rapid7/recog/verify/VerifyReporter.java
@@ -51,18 +51,18 @@ public void success(String text) {
}
}
- public void warning(String text) {
+ public void warning(String text, int line) {
if (!options.isWarnings()) {
return;
}
warningCount++;
- formatter.warningMessage(String.format("%s%s%s", pathLabel(), padding(), text));
+ formatter.warningMessage(String.format("%s%sWARN: %s", pathLabel(line), padding(), text));
}
- public void failure(String text) {
+ public void failure(String text, int line) {
failureCount++;
- formatter.failureMessage(String.format("%s%s%s", pathLabel(), padding(), text));
+ formatter.failureMessage(String.format("%s%sFAIL: %s", pathLabel(line), padding(), text));
}
public void printPath() {
@@ -99,8 +99,9 @@ private void resetCounts() {
warningCount = 0;
}
- private String pathLabel() {
- return options.isDetail() || path == null || path.isEmpty() ? "" : String.format("%s: ", path);
+ private String pathLabel(int line) {
+ String lineLabel = line > -1 ? String.format("%d:", line) : "";
+ return options.isDetail() || path == null || path.isEmpty() ? "" : String.format("%s:%s ", path, lineLabel);
}
private String padding() {
@@ -112,7 +113,7 @@ private String padding() {
private String summaryLine() {
return String.format("%sSUMMARY: Test completed with %d successful, %d warnings"
- + ", and %d failures", pathLabel(), successCount, warningCount, failureCount);
+ + ", and %d failures", pathLabel(-1), successCount, warningCount, failureCount);
}
private void colorizeSummary(String summary) {
diff --git a/recog/src/main/java/com/rapid7/recog/RecogMatcher.java b/recog/src/main/java/com/rapid7/recog/RecogMatcher.java
index db532aa..3be9acc 100644
--- a/recog/src/main/java/com/rapid7/recog/RecogMatcher.java
+++ b/recog/src/main/java/com/rapid7/recog/RecogMatcher.java
@@ -98,6 +98,9 @@ public static Map interpolate(String keyEndsWith, Map examples;
+ /** The matcher source data line number. */
+ private int line;
+
/**
* Creates a new RecogMatcher using a {@link JavaRegexRecogPatternMatcher} to
* match fingerprint values.
@@ -119,6 +122,7 @@ public RecogMatcher(RecogPatternMatcher matcher) {
positionalParameters = new LinkedHashMap<>();
namedParameters = new HashSet<>();
examples = new LinkedHashSet<>();
+ line = -1;
}
/**
@@ -142,6 +146,14 @@ public String getDescription() {
return description;
}
+ public int getLine() {
+ return line;
+ }
+
+ public void setLine(int line) {
+ this.line = line;
+ }
+
/**
* Adds an example to this matcher.
*
diff --git a/recog/src/main/java/com/rapid7/recog/parser/FingerprintsHandler.java b/recog/src/main/java/com/rapid7/recog/parser/FingerprintsHandler.java
new file mode 100644
index 0000000..9bfb0b2
--- /dev/null
+++ b/recog/src/main/java/com/rapid7/recog/parser/FingerprintsHandler.java
@@ -0,0 +1,270 @@
+package com.rapid7.recog.parser;
+
+import com.rapid7.recog.FingerprintExample;
+import com.rapid7.recog.RecogMatcher;
+import com.rapid7.recog.RecogMatchers;
+import com.rapid7.recog.parser.RecogParser.PatternMatcherFactory;
+import com.rapid7.recog.pattern.RecogPatternMatcher;
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.StringTokenizer;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.Attributes;
+import org.xml.sax.Locator;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+/**
+ * SAX2 event handler used to parse {@link RecogMatchers} from the XML content.
+ */
+public class FingerprintsHandler extends DefaultHandler {
+ private static final String FINGERPRINTS = "fingerprints";
+ private static final String FINGERPRINT = "fingerprint";
+ private static final String DESCRIPTION = "description";
+ private static final String EXAMPLE = "example";
+ private static final String PARAM = "param";
+ private static final String FILENAME_KEY = "_filename";
+ private static final Logger LOGGER = LoggerFactory.getLogger(FingerprintsHandler.class);
+
+ private final PatternMatcherFactory patternMatcherFactory;
+ private final boolean strictMode;
+ private final String path;
+ private final String name;
+
+ private Locator locator;
+ private RecogMatchers matchers;
+ private RecogMatcher fingerprintPattern;
+ private final StringBuilder elementValue;
+ private HashMap exampleAttributeMap;
+
+ /**
+ * Constructs a FingerprintsHandler.
+ *
+ * @param patternMatcherFactory Factory used to create the underlying {@link RecogPatternMatcher}.
+ * @param strictMode {@code true} if the parser should throw exceptions when any error is
+ * encountered, {@code false} otherwise.
+ * @param path Optional XML content file path.
+ * @param name Value used for {@link RecogMatchers} key if parsed value is null or empty.
+ */
+ public FingerprintsHandler(PatternMatcherFactory patternMatcherFactory, boolean strictMode, String path, String name) {
+ super();
+ this.patternMatcherFactory = patternMatcherFactory;
+ this.strictMode = strictMode;
+ this.path = path;
+ this.name = name;
+ this.elementValue = new StringBuilder();
+ }
+
+ /**
+ * Gets parsed {@link RecogMatchers}.
+ * @return parsed {@link RecogMatchers}.
+ */
+ public RecogMatchers getMatchers() {
+ return matchers;
+ }
+
+ @Override
+ public void setDocumentLocator(Locator locator) {
+ this.locator = locator; //Save the locator, so that it can be used later for line tracking when traversing nodes.
+ }
+
+ @Override
+ public void characters(char[] ch, int start, int length) throws SAXException {
+ elementValue.append(ch, start, length);
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
+ elementValue.setLength(0);
+ try {
+ switch (qName.toLowerCase()) {
+ case FINGERPRINTS:
+ String preferenceValue = getAttribute(attributes, "preference");
+ float preference = 0;
+ try {
+ preference = preferenceValue == null || preferenceValue.isEmpty() ? 0 : Float.parseFloat(preferenceValue);
+ } catch (NumberFormatException exception) {
+ // ignore - use default
+ }
+
+ String recogKey = getAttribute(attributes, "matches");
+
+ if (recogKey == null || recogKey.isEmpty()) {
+ LOGGER.debug("Recog Matcher Key is Empty or Null. File Name: " + name);
+ recogKey = name;
+ }
+
+ matchers = new RecogMatchers(path, recogKey, getAttribute(attributes, "protocol"), getAttribute(attributes, "database_type"), preference);
+ break;
+ case FINGERPRINT:
+ // the pattern is required
+ String pattern = getRequiredAttribute(attributes, "pattern");
+
+ // parse the flags for the regular expression
+ int regexFlags = parseFlags(getAttribute(attributes,"flags"));
+
+ // construct a pattern
+ fingerprintPattern = new RecogMatcher(patternMatcherFactory.create(pattern, regexFlags));
+ fingerprintPattern.setLine(locator.getLineNumber());
+ break;
+ case DESCRIPTION:
+ // NOP
+ break;
+ case EXAMPLE:
+ // example (optional)
+ exampleAttributeMap = new HashMap<>();
+ for (int i = 0; i < attributes.getLength(); i++) {
+ String attrName = attributes.getQName(i);
+ String attrValue = getAttribute(attributes, i);
+ exampleAttributeMap.put(attrName, attrValue);
+ }
+ break;
+ case PARAM:
+ if (fingerprintPattern == null) {
+ break;
+ }
+
+ int position = Integer.parseInt(getRequiredAttribute(attributes, "pos"));
+ String paramName = getRequiredAttribute(attributes, "name");
+
+ // zero position indicates a "constant" value
+ if (position == 0) {
+ String paramValue = getRequiredAttribute(attributes, "value");
+ fingerprintPattern.addValue(paramName, paramValue);
+ } else {
+ // otherwise the position indicates a group match result
+ String value = getAttribute(attributes, "value");
+ if (!value.isEmpty()) {
+ throw new ParseException(String.format("Attribute \"%s\" has a non-zero position but specifies a value of \"%s\"", paramName, value));
+ }
+ fingerprintPattern.addParam(position, paramName);
+ }
+ break;
+ default:
+ LOGGER.info("Unknown qualified name '{}'", qName);
+ }
+ } catch (ParseException | IllegalArgumentException exception) {
+ LOGGER.warn("Failed to parse fingerprint.", exception);
+ if (strictMode) {
+ throw exception;
+ }
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String qName) throws SAXException {
+ try {
+ switch (qName.toLowerCase()) {
+ case FINGERPRINTS:
+ case PARAM:
+ // NOP
+ break;
+ case FINGERPRINT:
+ if (fingerprintPattern == null) {
+ break;
+ }
+ matchers.add(fingerprintPattern);
+ break;
+ case DESCRIPTION:
+ // description (optional)
+ if (fingerprintPattern != null && elementValue.length() > 0) {
+ fingerprintPattern.setDescription(elementValue.toString().replaceAll("\\s+", " ").trim());
+ }
+ break;
+ case EXAMPLE:
+ if (fingerprintPattern == null) {
+ break;
+ }
+
+ String exampleText;
+ if (exampleAttributeMap != null && exampleAttributeMap.containsKey(FILENAME_KEY)) {
+ // process external example file
+ String filename = exampleAttributeMap.get(FILENAME_KEY);
+ exampleText = getExternalExampleText(path, name, filename);
+ } else {
+ exampleText = elementValue.toString();
+ }
+ fingerprintPattern.addExample(new FingerprintExample(exampleText, exampleAttributeMap));
+ break;
+ default:
+ LOGGER.info("Unknown qualified name '{}'", qName);
+ }
+ } catch (ParseException | IllegalArgumentException exception) {
+ LOGGER.warn("Failed to parse fingerprint.", exception);
+ if (strictMode) {
+ throw exception;
+ }
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////
+ // Non-public methods
+ /////////////////////////////////////////////////////////////////////////
+
+ private int parseFlags(String flags) {
+ int cflags = Pattern.UNIX_LINES;
+ if (flags != null && flags.length() != 0) {
+ StringTokenizer tok = new StringTokenizer(flags, "|,; \t");
+ while (tok.hasMoreTokens()) {
+ switch (tok.nextToken()) {
+ case "REG_ICASE":
+ case "IGNORECASE":
+ cflags |= Pattern.CASE_INSENSITIVE;
+ break;
+ case "REG_DOT_NEWLINE":
+ case "REG_MULTILINE":
+ cflags |= Pattern.DOTALL;
+ cflags |= Pattern.MULTILINE;
+ break;
+ default:
+ // ignore any other flags
+ break;
+ }
+ }
+ }
+
+ return cflags;
+ }
+
+ public String getAttribute(Attributes attr, String name) {
+ String value = attr.getValue(name);
+ return (value == null) ? "" : value;
+ }
+
+ public String getAttribute(Attributes attr, int index) {
+ String value = attr.getValue(index);
+ return (value == null) ? "" : value;
+ }
+
+ private String getRequiredAttribute(Attributes attr, String name) throws ParseException {
+ String value = getAttribute(attr, name);
+ if (value.length() == 0)
+ throw new ParseException(String.format("Attribute \"%s\" does not exist.", name));
+
+ return value;
+ }
+
+ private String getExternalExampleText(String path, String name, String filename) throws ParseException {
+ Path examplePath;
+ if (path != null) {
+ examplePath = Paths.get(Paths.get(path).getParent().toString(), name, filename);
+ } else {
+ examplePath = Paths.get(name, filename);
+ }
+
+ byte[] exampleBytes;
+ try {
+ exampleBytes = Files.readAllBytes(examplePath);
+ } catch (IOException exception) {
+ throw new ParseException(String.format("Unable to process fingerprint example file '%s'", examplePath), exception);
+ }
+ return new String(exampleBytes, StandardCharsets.US_ASCII);
+ }
+}
diff --git a/recog/src/main/java/com/rapid7/recog/parser/ParseException.java b/recog/src/main/java/com/rapid7/recog/parser/ParseException.java
index 6066e6f..870eb5c 100644
--- a/recog/src/main/java/com/rapid7/recog/parser/ParseException.java
+++ b/recog/src/main/java/com/rapid7/recog/parser/ParseException.java
@@ -1,15 +1,17 @@
package com.rapid7.recog.parser;
+import org.xml.sax.SAXException;
+
/**
* Thrown when an error parsing recog input occurs.
*/
-public class ParseException extends Exception {
+public class ParseException extends SAXException {
public ParseException(String message) {
super(message);
}
- public ParseException(String message, Throwable cause) {
+ public ParseException(String message, Exception cause) {
super(message, cause);
}
}
diff --git a/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java b/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java
index d1dcc92..9b93531 100644
--- a/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java
+++ b/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java
@@ -1,6 +1,5 @@
package com.rapid7.recog.parser;
-import com.rapid7.recog.FingerprintExample;
import com.rapid7.recog.RecogMatcher;
import com.rapid7.recog.RecogMatchers;
import com.rapid7.recog.pattern.JavaRegexRecogPatternMatcher;
@@ -9,25 +8,16 @@
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.StringTokenizer;
import java.util.regex.Pattern;
-import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
+
/**
* Parses {@link RecogMatcher} objects from XML input. This parser has support for strict or lenient
* parsing mode. In lenient mode parsing is tolerant and imposes the minimal constraints on input
@@ -36,6 +26,7 @@
* halt processing.
*/
public class RecogParser {
+
private static final String FILENAME_KEY = "_filename";
/**
@@ -127,167 +118,26 @@ public RecogMatchers parse(Reader reader, String name)
* if no matchers are defined, or all matchers are invalid and strict mode is disabled.
* @throws ParseException If an error is encountered and strict-mode is enabled.
*/
- public RecogMatchers parse(Reader reader, String path, String name)
- throws ParseException {
- Document document;
- try {
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- dbFactory.setAttribute("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
- dbFactory.setFeature("http://xml.org/sax/features/validation", false);
- dbFactory.setFeature("http://xml.org/sax/features/external-general-entities", false);
- dbFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
- document = dbFactory.newDocumentBuilder().parse(new InputSource(reader));
- } catch (SAXException | IOException | IllegalArgumentException | ParserConfigurationException exception) {
- throw new ParseException("Unable to parse fingerprints from Document", exception);
- }
-
- Element root = document.getDocumentElement();
-
- String preferenceValue = root.getAttribute("preference");
- float preference = 0;
+ public RecogMatchers parse(Reader reader, String path, String name) throws ParseException {
+ RecogMatchers matchers = null;
try {
- preference = preferenceValue == null || preferenceValue.isEmpty() ? 0 : Float.parseFloat(preferenceValue);
- } catch (NumberFormatException exception) {
- // ignore - use default
- }
-
- String recogKey = root.getAttribute("matches");
-
- if (recogKey.isEmpty()) {
- LOGGER.debug("Recog Matcher Key is Empty or Null. File Name: " + name);
- recogKey = name;
- }
-
- RecogMatchers matchers = new RecogMatchers(path, recogKey, root.getAttribute("protocol"), root.getAttribute("database_type"), preference);
-
- NodeList fingerprints = root.getElementsByTagName("fingerprint");
- for (int index = 0; index < fingerprints.getLength(); index++) {
- Element fingerprint = (Element) fingerprints.item(index);
- try {
- // the pattern is required
- String pattern = getRequiredAttribute(fingerprint, "pattern");
-
- // parse the flags for the regular expression
- int regexFlags = parseFlags(fingerprint.getAttribute("flags"));
-
- // construct a pattern
- RecogMatcher fingerprintPattern = new RecogMatcher(patternMatcherFactory.create(pattern, regexFlags));
-
- // description (optional)
- NodeList description = fingerprint.getElementsByTagName("description");
- if (description.getLength() > 0)
- fingerprintPattern.setDescription(description.item(0).getTextContent().replaceAll("\\s+", " ").trim());
-
- // example (optional)
- NodeList examples = fingerprint.getElementsByTagName("example");
- for (int examplesIndex = 0; examplesIndex < examples.getLength(); examplesIndex++) {
- Element example = (Element) examples.item(examplesIndex);
-
- HashMap attributeMap = new HashMap<>();
- NamedNodeMap exAttributes = example.getAttributes();
-
- for (int i = 0; i < exAttributes.getLength(); i++) {
- Node attr = exAttributes.item(i);
- String attrName = attr.getNodeName();
- String attrValue = attr.getNodeValue();
- attributeMap.put(attrName, attrValue);
- }
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+ factory.setFeature("http://xml.org/sax/features/validation", false);
+ factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
+ factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+ SAXParser saxParser = factory.newSAXParser();
+ FingerprintsHandler handler = new FingerprintsHandler(this.patternMatcherFactory, this.strictMode, path, name);
+ saxParser.parse(new InputSource(reader), handler);
+ matchers = handler.getMatchers();
+ } catch (ParseException exception) {
+ // re-throw ParseException
+ throw exception;
+ } catch (SAXException | IOException | ParserConfigurationException exception) {
+ System.out.printf("parse(): exception.getMessage(): %s\n", exception.getMessage());
- String exampleText;
- if (attributeMap.containsKey(FILENAME_KEY)) {
- // process external example file
- String filename = attributeMap.get(FILENAME_KEY);
- exampleText = getExternalExampleText(path, name, filename);
- } else {
- exampleText = example.getTextContent();
- }
- fingerprintPattern.addExample(new FingerprintExample(exampleText, attributeMap));
- }
-
- // parse and add parameter specifications
- NodeList params = fingerprint.getElementsByTagName("param");
- for (int paramIndex = 0; paramIndex < params.getLength(); paramIndex++) {
- Element parameter = (Element) params.item(paramIndex);
- int position = Integer.parseInt(getRequiredAttribute(parameter, "pos"));
-
- String paramName = getRequiredAttribute(parameter, "name");
-
- // zero position indicates a "constant" value
- if (position == 0) {
- String paramValue = getRequiredAttribute(parameter, "value");
- fingerprintPattern.addValue(paramName, paramValue);
- } else {
- // otherwise the position indicates a group match result
- String value = parameter.getAttribute("value");
- if (!value.isEmpty()) {
- throw new ParseException(String.format("Attribute \"%s\" has a non-zero position but specifies a value of \"%s\"", paramName, value));
- }
- fingerprintPattern.addParam(position, paramName);
- }
- }
-
- matchers.add(fingerprintPattern);
- } catch (ParseException | IllegalArgumentException exception) {
- LOGGER.warn("Failed to parse fingerprint.", exception);
- if (strictMode)
- throw exception;
- }
+ throw new ParseException("Unable to parse fingerprints from Document", exception);
}
-
return matchers;
}
-
- /////////////////////////////////////////////////////////////////////////
- // Non-public methods
- /////////////////////////////////////////////////////////////////////////
-
- private int parseFlags(String flags) {
- int cflags = Pattern.UNIX_LINES;
- if (flags != null && flags.length() != 0) {
- StringTokenizer tok = new StringTokenizer(flags, "|,; \t");
- while (tok.hasMoreTokens()) {
- switch (tok.nextToken()) {
- case "REG_ICASE":
- case "IGNORECASE":
- cflags |= Pattern.CASE_INSENSITIVE;
- break;
- case "REG_DOT_NEWLINE":
- case "REG_MULTILINE":
- cflags |= Pattern.DOTALL;
- cflags |= Pattern.MULTILINE;
- break;
- default:
- // ignore any other flags
- break;
- }
- }
- }
-
- return cflags;
- }
-
- private String getRequiredAttribute(Element element, String name) throws ParseException {
- String value = element.getAttribute(name);
- if (value.length() == 0)
- throw new ParseException("Attribute \"" + name + "\" does not exist.");
-
- return value;
- }
-
- private String getExternalExampleText(String path, String name, String filename) throws ParseException {
- Path examplePath;
- if (path != null) {
- examplePath = Paths.get(Paths.get(path).getParent().toString(), name, filename);
- } else {
- examplePath = Paths.get(name, filename);
- }
-
- byte[] exampleBytes;
- try {
- exampleBytes = Files.readAllBytes(examplePath);
- } catch (IOException exception) {
- throw new ParseException(String.format("Unable to process fingerprint example file '%s'", examplePath), exception);
- }
- return new String(exampleBytes, StandardCharsets.US_ASCII);
- }
}