diff --git a/metron-deployment/roles/opentaxii/templates/collections.yml b/metron-deployment/roles/opentaxii/templates/collections.yml index 986c6731e7..07a0e5afe4 100644 --- a/metron-deployment/roles/opentaxii/templates/collections.yml +++ b/metron-deployment/roles/opentaxii/templates/collections.yml @@ -15,4 +15,5 @@ # limitations under the License. # --- +collections: # intentionally blank - managed by ansible diff --git a/metron-deployment/roles/opentaxii/templates/services.yml b/metron-deployment/roles/opentaxii/templates/services.yml index 90c162ec01..61a1d42537 100644 --- a/metron-deployment/roles/opentaxii/templates/services.yml +++ b/metron-deployment/roles/opentaxii/templates/services.yml @@ -15,54 +15,55 @@ # limitations under the License. # --- -# -# discovery: used by a TAXII Client to discover available TAXII Service -# -- id: discovery - type: discovery - address: /services/discovery - description: Discovery service for Apache Metron - advertised_services: - - inbox - - collection - - poll - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# inbox: used by a TAXII Client to push information to a TAXII Server -# -- id: inbox - type: inbox - address: /services/inbox - description: Inbox for Apache Metron - destination_collection_required: yes - accept_all_content: yes - supported_content: - - urn:stix.mitre.org:xml:1.1.1 - authentication_required: no - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# collection_management: used by a TAXII Client to request information about -# available data collections or request a subscription. -# -- id: collection - type: collection_management - address: /services/collection - description: Collection management service for Apache Metron - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# poll: used by a TAXII Client to request information from a TAXII Server -# -- id: poll - type: poll - address: /services/poll - description: Poll service for Apache Metron - subscription_required: no - authentication_required: no - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 +services: + # + # discovery: used by a TAXII Client to discover available TAXII Service + # + - id: discovery + type: discovery + address: /services/discovery + description: Discovery service for Apache Metron + advertised_services: + - inbox + - collection + - poll + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # inbox: used by a TAXII Client to push information to a TAXII Server + # + - id: inbox + type: inbox + address: /services/inbox + description: Inbox for Apache Metron + destination_collection_required: yes + accept_all_content: yes + supported_content: + - urn:stix.mitre.org:xml:1.1.1 + authentication_required: no + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # collection_management: used by a TAXII Client to request information about + # available data collections or request a subscription. + # + - id: collection + type: collection_management + address: /services/collection + description: Collection management service for Apache Metron + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # poll: used by a TAXII Client to request information from a TAXII Server + # + - id: poll + type: poll + address: /services/poll + description: Poll service for Apache Metron + subscription_required: no + authentication_required: no + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java index bf42760aa3..1e1c0372a5 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java @@ -39,4 +39,8 @@ public Iterable extract(String line) throws IOException { public void initialize(Map config) { decoratedExtractor.initialize(config); } + + public Extractor getUnderlyingExtractor() { + return decoratedExtractor; + } } diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java index 516c31ee47..e2671e778a 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java @@ -60,7 +60,7 @@ public String toString() { } public boolean existsIn(Map config) { - return config.containsKey(key); + return config == null?false:config.containsKey(key); } } diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java index 689a08fce1..de6ae8d648 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java @@ -29,7 +29,9 @@ import org.apache.hadoop.util.GenericOptionsParser; import org.apache.log4j.PropertyConfigurator; import org.apache.metron.dataloads.extractor.Extractor; +import org.apache.metron.dataloads.extractor.ExtractorDecorator; import org.apache.metron.dataloads.extractor.ExtractorHandler; +import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator; import org.apache.metron.dataloads.extractor.stix.StixExtractor; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig; import org.apache.metron.common.utils.JSONUtils; @@ -165,6 +167,16 @@ public static Options getOptions() { public static final long DEFAULT_TIME_BETWEEN_POLLS = ONE_HR_IN_MS; + public static boolean isStixExtractor(Extractor e) { + if(e instanceof StixExtractor || + ( e instanceof ExtractorDecorator && ((ExtractorDecorator) e).getUnderlyingExtractor() instanceof StixExtractor) + ) { + return true; + } else { + return false; + } + } + public static void main(String... argv) throws Exception { Configuration conf = HBaseConfiguration.create(); String zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM); @@ -185,8 +197,8 @@ public static void main(String... argv) throws Exception { } Timer timer = new Timer(); - if(e instanceof StixExtractor) { - StixExtractor extractor = (StixExtractor)e; + if(isStixExtractor(e)) { + Extractor extractor = e; TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli)))); if(TaxiiOptions.BEGIN_TIME.has(cli)) { Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli)); diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java index 0223514bfa..7d077a3df6 100644 --- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java +++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java @@ -27,6 +27,8 @@ import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.metron.dataloads.extractor.Extractor; +import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator; import org.apache.metron.dataloads.extractor.stix.StixExtractor; import org.apache.metron.enrichment.converter.EnrichmentConverter; import org.apache.metron.enrichment.converter.EnrichmentKey; @@ -93,7 +95,8 @@ public void testTaxii() throws Exception { final MockHTable.Provider provider = new MockHTable.Provider(); final Configuration config = HBaseConfiguration.create(); - TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), new StixExtractor(), config ) { + Extractor extractor = new TransformFilterExtractorDecorator(new StixExtractor()); + TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), extractor, config ) { @Override protected synchronized HTableInterface createHTable(String tableInfo) throws IOException { return provider.addToCache("threat_intel", "cf");