METRON-1026: threatintel_taxii_load.sh throws exception

metron-deployment/roles/opentaxii/templates/collections.yml

@@ -15,4 +15,5 @@
 #  limitations under the License.
 #
 ---
+collections:
 # intentionally blank - managed by ansible

metron-deployment/roles/opentaxii/templates/services.yml

@@ -15,54 +15,55 @@
 #  limitations under the License.
 #
 ---
-#
-# discovery: used by a TAXII Client to discover available TAXII Service
-#
-- id: discovery
-  type: discovery
-  address: /services/discovery
-  description: Discovery service for Apache Metron
-  advertised_services:
-    - inbox
-    - collection
-    - poll
-  protocol_bindings:
-    - urn:taxii.mitre.org:protocol:http:1.0
-
-#
-# inbox: used by a TAXII Client to push information to a TAXII Server
-#
-- id: inbox
-  type: inbox
-  address: /services/inbox
-  description: Inbox for Apache Metron
-  destination_collection_required: yes
-  accept_all_content: yes
-  supported_content:
-    - urn:stix.mitre.org:xml:1.1.1
-  authentication_required: no
-  protocol_bindings:
-    - urn:taxii.mitre.org:protocol:http:1.0
-
-#
-# collection_management: used by a TAXII Client to request information about
-# available data collections or request a subscription.
-#
-- id: collection
-  type: collection_management
-  address: /services/collection
-  description: Collection management service for Apache Metron
-  protocol_bindings:
-    - urn:taxii.mitre.org:protocol:http:1.0
-
-#
-# poll: used by a TAXII Client to request information from a TAXII Server
-#
-- id: poll
-  type: poll
-  address: /services/poll
-  description: Poll service for Apache Metron
-  subscription_required: no
-  authentication_required: no
-  protocol_bindings:
-    - urn:taxii.mitre.org:protocol:http:1.0
+services:
+  #
+  # discovery: used by a TAXII Client to discover available TAXII Service
+  #
+  - id: discovery
+    type: discovery
+    address: /services/discovery
+    description: Discovery service for Apache Metron
+    advertised_services:
+      - inbox
+      - collection
+      - poll
+    protocol_bindings:
+      - urn:taxii.mitre.org:protocol:http:1.0
+
+  #
+  # inbox: used by a TAXII Client to push information to a TAXII Server
+  #
+  - id: inbox
+    type: inbox
+    address: /services/inbox
+    description: Inbox for Apache Metron
+    destination_collection_required: yes
+    accept_all_content: yes
+    supported_content:
+      - urn:stix.mitre.org:xml:1.1.1
+    authentication_required: no
+    protocol_bindings:
+      - urn:taxii.mitre.org:protocol:http:1.0
+
+  #
+  # collection_management: used by a TAXII Client to request information about
+  # available data collections or request a subscription.
+  #
+  - id: collection
+    type: collection_management
+    address: /services/collection
+    description: Collection management service for Apache Metron
+    protocol_bindings:
+      - urn:taxii.mitre.org:protocol:http:1.0
+
+  #
+  # poll: used by a TAXII Client to request information from a TAXII Server
+  #
+  - id: poll
+    type: poll
+    address: /services/poll
+    description: Poll service for Apache Metron
+    subscription_required: no
+    authentication_required: no
+    protocol_bindings:
+      - urn:taxii.mitre.org:protocol:http:1.0

metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java

@@ -39,4 +39,8 @@ public Iterable<LookupKV> extract(String line) throws IOException {
   public void initialize(Map<String, Object> config) {
     decoratedExtractor.initialize(config);
   }
+
+  public Extractor getUnderlyingExtractor() {
+    return decoratedExtractor;
+  }
 }

metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java

@@ -60,7 +60,7 @@ public String toString() {
     }
 
     public boolean existsIn(Map<String, Object> config) {
-      return config.containsKey(key);
+      return config == null?false:config.containsKey(key);
     }
   }
 

metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java

@@ -29,7 +29,9 @@
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.log4j.PropertyConfigurator;
 import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.dataloads.extractor.ExtractorDecorator;
 import org.apache.metron.dataloads.extractor.ExtractorHandler;
+import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator;
 import org.apache.metron.dataloads.extractor.stix.StixExtractor;
 import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig;
 import org.apache.metron.common.utils.JSONUtils;
@@ -165,6 +167,16 @@ public static Options getOptions() {
   public static final long DEFAULT_TIME_BETWEEN_POLLS = ONE_HR_IN_MS;
 
 
+  public static boolean isStixExtractor(Extractor e) {
+    if(e instanceof StixExtractor ||
+       ( e instanceof ExtractorDecorator && ((ExtractorDecorator) e).getUnderlyingExtractor() instanceof StixExtractor)
+      ) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
   public static void main(String... argv) throws Exception {
     Configuration conf = HBaseConfiguration.create();
     String zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM);
@@ -185,8 +197,8 @@ public static void main(String... argv) throws Exception {
     }
 
     Timer timer = new Timer();
-    if(e instanceof StixExtractor) {
-      StixExtractor extractor = (StixExtractor)e;
+    if(isStixExtractor(e)) {
+      Extractor extractor = e;
       TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli))));
       if(TaxiiOptions.BEGIN_TIME.has(cli)) {
         Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli));

metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java

@@ -27,6 +27,8 @@
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator;
 import org.apache.metron.dataloads.extractor.stix.StixExtractor;
 import org.apache.metron.enrichment.converter.EnrichmentConverter;
 import org.apache.metron.enrichment.converter.EnrichmentKey;
@@ -93,7 +95,8 @@ public void testTaxii() throws Exception {
 
         final MockHTable.Provider provider = new MockHTable.Provider();
         final Configuration config = HBaseConfiguration.create();
-        TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), new StixExtractor(), config ) {
+        Extractor extractor = new TransformFilterExtractorDecorator(new StixExtractor());
+        TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), extractor, config ) {
             @Override
             protected synchronized HTableInterface createHTable(String tableInfo) throws IOException {
                 return provider.addToCache("threat_intel", "cf");