diff --git a/metron-platform/metron-data-management/README.md b/metron-platform/metron-data-management/README.md index 014f3efca1..c351f48e29 100644 --- a/metron-platform/metron-data-management/README.md +++ b/metron-platform/metron-data-management/README.md @@ -97,6 +97,11 @@ Consider the following config for importing STIX documents. This is a threat in format, so it is particularly relevant and attractive data to import for our purposes. Because STIX is a standard format, there is no need to specify the schema or how to interpret the documents. +We support the versions of Stix and Cybox supported by +[java-stix](https://github.com/STIXProject/java-stix/tree/v1.2.0.2): +* Stix - [1.2](https://github.com/STIXProject/schemas/blob/356cc4f6b06625465f0808388eb166807313b4e0/stix_core.xsd) and earlier +* Cybox - [2.1](https://github.com/CybOXProject/schemas/blob/97beb32c376a9223e91b52cb3e4c8d2af6baf786/cybox_core.xsd) and earlier + We support a subset of STIX messages for importation: | STIX Type | Specific Type | Enrichment Type Name | @@ -107,6 +112,7 @@ We support a subset of STIX messages for importation: | Address | MAC | address:MAC | | Domain | FQDN | domain:FQDN | | Hostname | | hostname | +| URI | | uriobjecttype | NOTE: The enrichment type will be used as the type above. diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java index 41b428e97a..f96bd2d74c 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java @@ -30,14 +30,18 @@ import org.mitre.stix.common_1.IndicatorBaseType; import org.mitre.stix.indicator_2.Indicator; import org.mitre.stix.stix_1.STIXPackage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.Map; public class StixExtractor implements Extractor { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); Map config; @Override public Iterable extract(String line) throws IOException { @@ -50,11 +54,22 @@ public Iterable extract(String line) throws IOException { if(props != null) { ObjectTypeHandler handler = ObjectTypeHandlers.getHandlerByInstance(props); if (handler != null) { + if(LOG.isDebugEnabled()) { + LOG.debug("Found {} for properties {}" + , handler.getTypeClass().getCanonicalName() + , props.toXMLString()); + } Iterable extractions = handler.extract(props, config); for(LookupKV extraction : extractions) { ret.add(extraction); } } + else if(LOG.isDebugEnabled()) { + LOG.debug("Did not find a handler for properties {} of type {}" + , props.toXMLString() + , props.getClass() + ); + } } } } diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java index 06d8cd87b3..df2c91571a 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java @@ -23,6 +23,7 @@ public enum ObjectTypeHandlers { ADDRESS(new AddressHandler()) ,HOSTNAME(new HostnameHandler()) ,DOMAINNAME(new DomainHandler()) + ,URI(new URIHandler()) ,; ObjectTypeHandler _handler; ObjectTypeHandlers(ObjectTypeHandler handler) { diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java new file mode 100644 index 0000000000..2adfb8a9ff --- /dev/null +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.dataloads.extractor.stix.types; + +import com.google.common.collect.ImmutableList; +import org.apache.metron.enrichment.converter.EnrichmentKey; +import org.apache.metron.enrichment.converter.EnrichmentValue; +import org.apache.metron.enrichment.lookup.LookupKV; +import org.mitre.cybox.common_2.AnyURIObjectPropertyType; +import org.mitre.cybox.objects.URIObjectType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class URIHandler extends AbstractObjectTypeHandler { + public URIHandler() { + super(URIObjectType.class); + } + + @Override + public Iterable extract(URIObjectType type, Map config) throws IOException { + List ret = new ArrayList<>(); + if(type != null) { + AnyURIObjectPropertyType val = type.getValue(); + if(val != null) { + Object v = val.getValue(); + if(v != null) { + final String indicatorType = getType(); + LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, v.toString()) + , new EnrichmentValue( + new HashMap() {{ + put("source-type", "STIX"); + put("uri", v.toString()); + put("indicator-type", indicatorType); + put("source", type.toXMLString()); + }} + ) + ); + ret.add(results); + } + } + } + return ret; + } + + @Override + public List getPossibleTypes() { + return ImmutableList.of(getType()); + } +} diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java new file mode 100644 index 0000000000..ff41152b23 --- /dev/null +++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.dataloads.extractor.stix; + +import com.google.common.collect.Iterables; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.dataloads.extractor.stix.types.URIHandler; +import org.apache.metron.enrichment.converter.EnrichmentKey; +import org.apache.metron.enrichment.lookup.LookupKV; +import org.junit.Assert; +import org.junit.Test; +import org.mitre.cybox.objects.URIObjectType; + +import java.util.HashMap; +import java.util.List; + +public class URIHandlerTest { + + /** + * + * + * + * + * + * ../../../../descendant-or-self::node() + * + * + * TBD + * + * + * Unclassified (Public) + * + * + * + * + * + * + * URL: http://www.kotimi.com/alpha/gtex/... + * URL: http://www.kotimi.com/alpha/gtex/| isOnline:yes| dateVerified:2017-07-31T22:03:10+00:00 + * + * + * http://www.kotimi.com/alpha/gtex/ + * + * + * + * + * + */ + @Multiline + static String uriHandlerObject; + + @Test + public void testURIHandler() throws Exception { + StixExtractor extractor = new StixExtractor(); + extractor.initialize(new HashMap<>()); + Iterable kvs = extractor.extract(uriHandlerObject); + Assert.assertEquals(1, Iterables.size(kvs)); + LookupKV kv = Iterables.getFirst(kvs, null); + EnrichmentKey key = (EnrichmentKey) kv.getKey(); + Assert.assertEquals("http://www.kotimi.com/alpha/gtex/", key.getIndicator()); + Assert.assertEquals("uriobjecttype", key.type); + } +}