METRON-1102: Add support for ingesting cybox URI observables from taxii feeds

metron-platform/metron-data-management/README.md

@@ -97,6 +97,11 @@ Consider the following config for importing STIX documents.  This is a threat in
 format, so it is particularly relevant and attractive data to import for our purposes.  Because STIX is
 a standard format, there is no need to specify the schema or how to interpret the documents.
 
+We support the versions of Stix and Cybox supported by
+[java-stix](https://github.com/STIXProject/java-stix/tree/v1.2.0.2):
+* Stix - [1.2](https://github.com/STIXProject/schemas/blob/356cc4f6b06625465f0808388eb166807313b4e0/stix_core.xsd) and earlier
+* Cybox - [2.1](https://github.com/CybOXProject/schemas/blob/97beb32c376a9223e91b52cb3e4c8d2af6baf786/cybox_core.xsd) and earlier
+
 We support a subset of STIX messages for importation:
 
 | STIX Type | Specific Type | Enrichment Type Name |
@@ -107,6 +112,7 @@ We support a subset of STIX messages for importation:
 | Address   | MAC           | address:MAC          |
 | Domain    | FQDN          | domain:FQDN          |
 | Hostname  |               | hostname             |
+| URI       |               | uriobjecttype        |
 
 
 NOTE: The enrichment type will be used as the type above.

metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java

@@ -30,14 +30,18 @@
 import org.mitre.stix.common_1.IndicatorBaseType;
 import org.mitre.stix.indicator_2.Indicator;
 import org.mitre.stix.stix_1.STIXPackage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 public class StixExtractor implements Extractor {
+    private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
     Map<String, Object> config;
     @Override
     public Iterable<LookupKV> extract(String line) throws IOException {
@@ -50,11 +54,22 @@ public Iterable<LookupKV> extract(String line) throws IOException {
                 if(props != null) {
                     ObjectTypeHandler handler = ObjectTypeHandlers.getHandlerByInstance(props);
                     if (handler != null) {
+                        if(LOG.isDebugEnabled()) {
+                            LOG.debug("Found {} for properties {}"
+                                    , handler.getTypeClass().getCanonicalName()
+                                    , props.toXMLString());
+                        }
                         Iterable<LookupKV> extractions = handler.extract(props, config);
                         for(LookupKV extraction : extractions) {
                             ret.add(extraction);
                         }
                     }
+                    else if(LOG.isDebugEnabled()) {
+                        LOG.debug("Did not find a handler for properties {} of type {}"
+                                , props.toXMLString()
+                                , props.getClass()
+                        );
+                    }
                 }
             }
         }

metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java

@@ -23,6 +23,7 @@ public enum ObjectTypeHandlers {
       ADDRESS(new AddressHandler())
     ,HOSTNAME(new HostnameHandler())
     ,DOMAINNAME(new DomainHandler())
+    ,URI(new URIHandler())
     ,;
    ObjectTypeHandler _handler;
    ObjectTypeHandlers(ObjectTypeHandler handler) {

metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java

@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.dataloads.extractor.stix.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.lookup.LookupKV;
+import org.mitre.cybox.common_2.AnyURIObjectPropertyType;
+import org.mitre.cybox.objects.URIObjectType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class URIHandler extends AbstractObjectTypeHandler<URIObjectType> {
+  public URIHandler() {
+    super(URIObjectType.class);
+  }
+
+  @Override
+  public Iterable<LookupKV> extract(URIObjectType type, Map<String, Object> config) throws IOException {
+    List<LookupKV> ret = new ArrayList<>();
+    if(type != null) {
+      AnyURIObjectPropertyType val = type.getValue();
+      if(val != null) {
+        Object v = val.getValue();
+        if(v != null) {
+          final String indicatorType = getType();
+          LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, v.toString())
+                  , new EnrichmentValue(
+                  new HashMap<String, Object>() {{
+                    put("source-type", "STIX");
+                    put("uri", v.toString());
+                    put("indicator-type", indicatorType);
+                    put("source", type.toXMLString());
+                  }}
+          )
+          );
+          ret.add(results);
+        }
+      }
+    }
+    return ret;
+  }
+
+  @Override
+  public List<String> getPossibleTypes() {
+    return ImmutableList.of(getType());
+  }
+}

metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java

@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.dataloads.extractor.stix;
+
+import com.google.common.collect.Iterables;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.dataloads.extractor.stix.types.URIHandler;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.lookup.LookupKV;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mitre.cybox.objects.URIObjectType;
+
+import java.util.HashMap;
+import java.util.List;
+
+public class URIHandlerTest {
+
+  /**
+   *<?xml version="1.0" encoding="UTF-8"?>
+   *<stix:STIX_Package xmlns:stix="http://stix.mitre.org/stix-1"
+   *                   xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1"
+   *                   xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1"
+   *                   xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1"
+   *                   xmlns:cyboxCommon="http://cybox.mitre.org/common-2"
+   *                   xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1"
+   *                   xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2"
+   *                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+   *                   xmlns:stixCommon="http://stix.mitre.org/common-1"
+   *                   xmlns:edge="http://soltra.com/"
+   *                   xmlns:marking="http://data-marking.mitre.org/Marking-1"
+   *                   xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1"
+   *                   xmlns:taxii_11="http://taxii.mitre.org/messages/taxii_xml_binding-1.1"
+   *                   xmlns:opensource="http://hailataxii.com"
+   *                   xmlns:URIObj="http://cybox.mitre.org/objects#URIObject-2"
+   *                   xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1"
+   *                   xmlns:cybox="http://cybox.mitre.org/cybox-2"
+   *                   id="edge:Package-208ba7e1-ecc1-49a1-a96d-f28c4146761d"
+   *                   timestamp="2017-08-09T21:05:27.148461+00:00"
+   *                   version="1.1.1">
+   *  <stix:STIX_Header>
+   *    <stix:Handling>
+   *      <marking:Marking>
+   *        <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+   *        <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+   *        <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+   *          <TOUMarking:Terms_Of_Use>TBD</TOUMarking:Terms_Of_Use>
+   *        </marking:Marking_Structure>
+   *        <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+   *          <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+   *        </marking:Marking_Structure>
+   *      </marking:Marking>
+   *    </stix:Handling>
+   *  </stix:STIX_Header>
+   *  <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+   *    <cybox:Observable id="opensource:Observable-6b98960f-c8bb-45fd-8b6d-8960e803b51f" sighting_count="1">
+   *      <cybox:Title>URL: http://www.kotimi.com/alpha/gtex/...</cybox:Title>
+   *      <cybox:Description>URL: http://www.kotimi.com/alpha/gtex/| isOnline:yes| dateVerified:2017-07-31T22:03:10+00:00</cybox:Description>
+   *      <cybox:Object id="opensource:URI-9baf3b48-4aa2-4198-92b7-b5cb0a0a1d35">
+   *        <cybox:Properties type="URL" xsi:type="URIObj:URIObjectType">
+   *          <URIObj:Value condition="Equals">http://www.kotimi.com/alpha/gtex/</URIObj:Value>
+   *        </cybox:Properties>
+   *      </cybox:Object>
+   *    </cybox:Observable>
+   *  </stix:Observables>
+   *</stix:STIX_Package>
+   */
+  @Multiline
+  static String uriHandlerObject;
+
+  @Test
+  public void testURIHandler() throws Exception {
+    StixExtractor extractor = new StixExtractor();
+    extractor.initialize(new HashMap<>());
+    Iterable<LookupKV> kvs = extractor.extract(uriHandlerObject);
+    Assert.assertEquals(1, Iterables.size(kvs));
+    LookupKV kv = Iterables.getFirst(kvs, null);
+    EnrichmentKey key = (EnrichmentKey) kv.getKey();
+    Assert.assertEquals("http://www.kotimi.com/alpha/gtex/", key.getIndicator());
+    Assert.assertEquals("uriobjecttype", key.type);
+  }
+}