From 06fcd8047a6c2cbdc6b8d1ad7a432c867998f08d Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 10 Jul 2017 14:30:43 +0100 Subject: [PATCH 1/6] Corrected taxii issue. --- .../dataloads/extractor/ExtractorDecorator.java | 4 ++++ .../dataloads/nonbulk/taxii/TaxiiLoader.java | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java index bf42760aa3..1e1c0372a5 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java @@ -39,4 +39,8 @@ public Iterable extract(String line) throws IOException { public void initialize(Map config) { decoratedExtractor.initialize(config); } + + public Extractor getUnderlyingExtractor() { + return decoratedExtractor; + } } diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java index 689a08fce1..1cad106a13 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java @@ -29,7 +29,9 @@ import org.apache.hadoop.util.GenericOptionsParser; import org.apache.log4j.PropertyConfigurator; import org.apache.metron.dataloads.extractor.Extractor; +import org.apache.metron.dataloads.extractor.ExtractorDecorator; import org.apache.metron.dataloads.extractor.ExtractorHandler; +import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator; import org.apache.metron.dataloads.extractor.stix.StixExtractor; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig; import org.apache.metron.common.utils.JSONUtils; @@ -165,6 +167,19 @@ public static Options getOptions() { public static final long DEFAULT_TIME_BETWEEN_POLLS = ONE_HR_IN_MS; + public static boolean isStixExtractor(Extractor e) { + if(e instanceof StixExtractor) { + return true; + } + else if(e instanceof ExtractorDecorator) { + ExtractorDecorator decorator = (ExtractorDecorator)e; + if(decorator.getUnderlyingExtractor() != null && decorator.getUnderlyingExtractor() instanceof StixExtractor) { + return true; + } + } + return false; + } + public static void main(String... argv) throws Exception { Configuration conf = HBaseConfiguration.create(); String zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM); @@ -185,7 +200,7 @@ public static void main(String... argv) throws Exception { } Timer timer = new Timer(); - if(e instanceof StixExtractor) { + if(isStixExtractor(e)) { StixExtractor extractor = (StixExtractor)e; TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli)))); if(TaxiiOptions.BEGIN_TIME.has(cli)) { From dd3a05ec85e81eb9027f2d1ff39559c7e8ba5264 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 26 Jul 2017 11:10:07 +0100 Subject: [PATCH 2/6] Made a bit more readable. --- .../metron/dataloads/nonbulk/taxii/TaxiiLoader.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java index 1cad106a13..38d0cd6a73 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java @@ -168,16 +168,13 @@ public static Options getOptions() { public static boolean isStixExtractor(Extractor e) { - if(e instanceof StixExtractor) { + if(e instanceof StixExtractor || + ( e instanceof ExtractorDecorator && ((ExtractorDecorator) e).getUnderlyingExtractor() instanceof StixExtractor) + ) { return true; + } else { + return false; } - else if(e instanceof ExtractorDecorator) { - ExtractorDecorator decorator = (ExtractorDecorator)e; - if(decorator.getUnderlyingExtractor() != null && decorator.getUnderlyingExtractor() instanceof StixExtractor) { - return true; - } - } - return false; } public static void main(String... argv) throws Exception { From 3cc578a05a18319eda657a2afcb2e6f2aa860c2e Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 26 Jul 2017 11:27:23 +0100 Subject: [PATCH 3/6] Missed an unnecessary cast. --- .../org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java index 38d0cd6a73..de6ae8d648 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java @@ -198,7 +198,7 @@ public static void main(String... argv) throws Exception { Timer timer = new Timer(); if(isStixExtractor(e)) { - StixExtractor extractor = (StixExtractor)e; + Extractor extractor = e; TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli)))); if(TaxiiOptions.BEGIN_TIME.has(cli)) { Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli)); From 7aa7f11811209cff1260f5f753711e00e04f68fc Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 26 Jul 2017 11:29:59 +0100 Subject: [PATCH 4/6] Updating integration test to be more clear. --- .../metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java index 0223514bfa..7d077a3df6 100644 --- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java +++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java @@ -27,6 +27,8 @@ import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.metron.dataloads.extractor.Extractor; +import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator; import org.apache.metron.dataloads.extractor.stix.StixExtractor; import org.apache.metron.enrichment.converter.EnrichmentConverter; import org.apache.metron.enrichment.converter.EnrichmentKey; @@ -93,7 +95,8 @@ public void testTaxii() throws Exception { final MockHTable.Provider provider = new MockHTable.Provider(); final Configuration config = HBaseConfiguration.create(); - TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), new StixExtractor(), config ) { + Extractor extractor = new TransformFilterExtractorDecorator(new StixExtractor()); + TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), extractor, config ) { @Override protected synchronized HTableInterface createHTable(String tableInfo) throws IOException { return provider.addToCache("threat_intel", "cf"); From 8e326fc2570fe2c9dfbed36a7c9a6c2d5b8f4696 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 9 Aug 2017 16:42:59 -0400 Subject: [PATCH 5/6] Collections and services config for opentaxii changed. --- .../roles/opentaxii/templates/collections.yml | 1 + .../roles/opentaxii/templates/services.yml | 103 +++++++++--------- 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/metron-deployment/roles/opentaxii/templates/collections.yml b/metron-deployment/roles/opentaxii/templates/collections.yml index 986c6731e7..07a0e5afe4 100644 --- a/metron-deployment/roles/opentaxii/templates/collections.yml +++ b/metron-deployment/roles/opentaxii/templates/collections.yml @@ -15,4 +15,5 @@ # limitations under the License. # --- +collections: # intentionally blank - managed by ansible diff --git a/metron-deployment/roles/opentaxii/templates/services.yml b/metron-deployment/roles/opentaxii/templates/services.yml index 90c162ec01..61a1d42537 100644 --- a/metron-deployment/roles/opentaxii/templates/services.yml +++ b/metron-deployment/roles/opentaxii/templates/services.yml @@ -15,54 +15,55 @@ # limitations under the License. # --- -# -# discovery: used by a TAXII Client to discover available TAXII Service -# -- id: discovery - type: discovery - address: /services/discovery - description: Discovery service for Apache Metron - advertised_services: - - inbox - - collection - - poll - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# inbox: used by a TAXII Client to push information to a TAXII Server -# -- id: inbox - type: inbox - address: /services/inbox - description: Inbox for Apache Metron - destination_collection_required: yes - accept_all_content: yes - supported_content: - - urn:stix.mitre.org:xml:1.1.1 - authentication_required: no - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# collection_management: used by a TAXII Client to request information about -# available data collections or request a subscription. -# -- id: collection - type: collection_management - address: /services/collection - description: Collection management service for Apache Metron - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# poll: used by a TAXII Client to request information from a TAXII Server -# -- id: poll - type: poll - address: /services/poll - description: Poll service for Apache Metron - subscription_required: no - authentication_required: no - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 +services: + # + # discovery: used by a TAXII Client to discover available TAXII Service + # + - id: discovery + type: discovery + address: /services/discovery + description: Discovery service for Apache Metron + advertised_services: + - inbox + - collection + - poll + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # inbox: used by a TAXII Client to push information to a TAXII Server + # + - id: inbox + type: inbox + address: /services/inbox + description: Inbox for Apache Metron + destination_collection_required: yes + accept_all_content: yes + supported_content: + - urn:stix.mitre.org:xml:1.1.1 + authentication_required: no + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # collection_management: used by a TAXII Client to request information about + # available data collections or request a subscription. + # + - id: collection + type: collection_management + address: /services/collection + description: Collection management service for Apache Metron + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # poll: used by a TAXII Client to request information from a TAXII Server + # + - id: poll + type: poll + address: /services/poll + description: Poll service for Apache Metron + subscription_required: no + authentication_required: no + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 From affabb9d3e1d2375cccb9dd71874b007e64d3677 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 9 Aug 2017 17:16:02 -0400 Subject: [PATCH 6/6] Be more resilient here and null-check. --- .../dataloads/extractor/TransformFilterExtractorDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java index 516c31ee47..e2671e778a 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java @@ -60,7 +60,7 @@ public String toString() { } public boolean existsIn(Map config) { - return config.containsKey(key); + return config == null?false:config.containsKey(key); } }