From e7f00dec4d45b5dcea58c3d8dfdc9cc06db4aebd Mon Sep 17 00:00:00 2001 From: Matthew Kienow Date: Wed, 3 Nov 2021 17:47:24 -0400 Subject: [PATCH 1/3] Add support for base64 encoded examples --- .../com/rapid7/recog/FingerprintExample.java | 14 +++++++++-- .../com/rapid7/recog/parser/RecogParser.java | 23 ++++++++----------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/recog/src/main/java/com/rapid7/recog/FingerprintExample.java b/recog/src/main/java/com/rapid7/recog/FingerprintExample.java index 6f56e09..f655994 100644 --- a/recog/src/main/java/com/rapid7/recog/FingerprintExample.java +++ b/recog/src/main/java/com/rapid7/recog/FingerprintExample.java @@ -1,15 +1,25 @@ package com.rapid7.recog; +import java.util.Base64; import java.util.Map; +import static java.util.Objects.requireNonNull; // Represents a fingerprint example and associated data. public class FingerprintExample { + private static final String ENCODING_KEY = "_encoding"; + private final String text; private final Map attributeMap; public FingerprintExample(String text, Map attributeMap) { - this.text = text; - this.attributeMap = attributeMap; + String tmpText = requireNonNull(text); + this.attributeMap = requireNonNull(attributeMap); + if (attributeMap.containsKey(ENCODING_KEY) && attributeMap.get(ENCODING_KEY).equals("base64")) { + byte[] exampleContentBytes = Base64.getDecoder().decode(tmpText.replaceAll("\\s+", "")); + this.text = new String(exampleContentBytes); + } else { + this.text = text; + } } public String getText() { diff --git a/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java b/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java index 4ff8057..9d4bc5e 100644 --- a/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java +++ b/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java @@ -177,23 +177,18 @@ public RecogMatchers parse(Reader reader, String path, String name) NodeList examples = fingerprint.getElementsByTagName("example"); for (int examplesIndex = 0; examplesIndex < examples.getLength(); examplesIndex++) { Element example = (Element) examples.item(examplesIndex); - String exampleContent = example.getTextContent(); - if ("base64".equals(example.getAttribute("_encoding"))) { - // TODO: these are currently ignored as the Base64 decoding isn't working properly - } else { - HashMap attributeMap = new HashMap<>(); - NamedNodeMap exAttributes = example.getAttributes(); - - for (int i = 0; i < exAttributes.getLength(); i++) { - Node attr = exAttributes.item(i); - String attrName = attr.getNodeName(); - String attrValue = attr.getNodeValue(); - attributeMap.put(attrName, attrValue); - } + HashMap attributeMap = new HashMap<>(); + NamedNodeMap exAttributes = example.getAttributes(); - fingerprintPattern.addExample(new FingerprintExample(exampleContent, attributeMap)); + for (int i = 0; i < exAttributes.getLength(); i++) { + Node attr = exAttributes.item(i); + String attrName = attr.getNodeName(); + String attrValue = attr.getNodeValue(); + attributeMap.put(attrName, attrValue); } + + fingerprintPattern.addExample(new FingerprintExample(example.getTextContent(), attributeMap)); } // parse and add parameter specifications From 689e9cf44f12acd834d5ea671ce51de635ffec3f Mon Sep 17 00:00:00 2001 From: Matthew Kienow Date: Wed, 3 Nov 2021 18:23:59 -0400 Subject: [PATCH 2/3] Modify pattern flags to increase Ruby similarity These changes helped to resolve most issues discovered when testing base64 encoded examples. --- recog/src/main/java/com/rapid7/recog/RecogMatcher.java | 2 +- recog/src/main/java/com/rapid7/recog/parser/RecogParser.java | 5 ++--- .../rapid7/recog/parser/FingerprintMatcherParserTest.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/recog/src/main/java/com/rapid7/recog/RecogMatcher.java b/recog/src/main/java/com/rapid7/recog/RecogMatcher.java index 442a81d..c41eda5 100644 --- a/recog/src/main/java/com/rapid7/recog/RecogMatcher.java +++ b/recog/src/main/java/com/rapid7/recog/RecogMatcher.java @@ -350,7 +350,7 @@ public String getPattern() { * @return A {@link Pattern} with the compiled flags provided. Will not be {@code null}. */ public static Pattern pattern(String regex, int... flags) { - int patternFlags = 0; + int patternFlags = Pattern.UNIX_LINES; for (int flag : flags) patternFlags |= flag; diff --git a/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java b/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java index 9d4bc5e..128010f 100644 --- a/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java +++ b/recog/src/main/java/com/rapid7/recog/parser/RecogParser.java @@ -229,7 +229,7 @@ public RecogMatchers parse(Reader reader, String path, String name) ///////////////////////////////////////////////////////////////////////// private int parseFlags(String flags) { - int cflags = 0; + int cflags = Pattern.UNIX_LINES; if (flags != null && flags.length() != 0) { StringTokenizer tok = new StringTokenizer(flags, "|,; \t"); while (tok.hasMoreTokens()) { @@ -239,9 +239,8 @@ private int parseFlags(String flags) { cflags |= Pattern.CASE_INSENSITIVE; break; case "REG_DOT_NEWLINE": - cflags |= Pattern.DOTALL; - break; case "REG_MULTILINE": + cflags |= Pattern.DOTALL; cflags |= Pattern.MULTILINE; break; default: diff --git a/recog/src/test/java/com/rapid7/recog/parser/FingerprintMatcherParserTest.java b/recog/src/test/java/com/rapid7/recog/parser/FingerprintMatcherParserTest.java index faee403..0852167 100644 --- a/recog/src/test/java/com/rapid7/recog/parser/FingerprintMatcherParserTest.java +++ b/recog/src/test/java/com/rapid7/recog/parser/FingerprintMatcherParserTest.java @@ -94,7 +94,7 @@ public void twoValidFingerprints() throws ParseException { assertThat(patterns.size(), is(2)); assertThat(patterns, hasItems( new RecogMatcher(pattern("^Apache/\\d$", CASE_INSENSITIVE)).addValue("service.vendor", "Apache").addValue("service.product", "HTTPD").addValue("service.family", "Apache"), - new RecogMatcher(pattern("^Apache$", MULTILINE)).addValue("service.vendor", "Apache").addValue("service.product", "HTTPD").addValue("service.family", "Apache"))); + new RecogMatcher(pattern("^Apache$", DOTALL, MULTILINE)).addValue("service.vendor", "Apache").addValue("service.product", "HTTPD").addValue("service.family", "Apache"))); } @Test From 19541bd7009f0a362ff03fd3559cb2912acab899 Mon Sep 17 00:00:00 2001 From: Matthew Kienow Date: Wed, 3 Nov 2021 18:42:20 -0400 Subject: [PATCH 3/3] Remove base64 encoding from missing features list --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 96e6359..ebdb376 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,6 @@ This library is not yet at a 1:1 parity with the original [rapid7/recog](https:/ Missing features: - Matching against multi-line input strings -- Matching against base64 encoded strings - Command line tools like `recog_match` ## Development