From 2d0e597c3fe8efec9e3ff5d5c98e949b70cddd85 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 9 Aug 2017 20:02:42 -0400 Subject: [PATCH 1/4] Add URI Stix type handler --- .../extractor/stix/StixExtractor.java | 12 +++ .../stix/types/ObjectTypeHandlers.java | 1 + .../extractor/stix/types/URIHandler.java | 68 +++++++++++++ .../extractor/stix/URIHandlerTest.java | 97 +++++++++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java create mode 100644 metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java index 41b428e97a..c926b79f1a 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java @@ -19,6 +19,7 @@ import com.google.common.base.Splitter; import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; import org.apache.metron.dataloads.extractor.Extractor; import org.apache.metron.dataloads.extractor.stix.types.ObjectTypeHandler; import org.apache.metron.dataloads.extractor.stix.types.ObjectTypeHandlers; @@ -38,6 +39,7 @@ import java.util.Map; public class StixExtractor implements Extractor { + private static final Logger LOG = Logger.getLogger(StixExtractor.class); Map config; @Override public Iterable extract(String line) throws IOException { @@ -50,11 +52,21 @@ public Iterable extract(String line) throws IOException { if(props != null) { ObjectTypeHandler handler = ObjectTypeHandlers.getHandlerByInstance(props); if (handler != null) { + if(LOG.isDebugEnabled()) { + LOG.debug("Found " + handler.getTypeClass().getCanonicalName() + + " for properties " + props.toXMLString() + ); + } Iterable extractions = handler.extract(props, config); for(LookupKV extraction : extractions) { ret.add(extraction); } } + else if(LOG.isDebugEnabled()) { + LOG.debug("Did not find a handler" + + " for properties " + props.toXMLString() + " of type " + props.getClass() + ); + } } } } diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java index 06d8cd87b3..df2c91571a 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java @@ -23,6 +23,7 @@ public enum ObjectTypeHandlers { ADDRESS(new AddressHandler()) ,HOSTNAME(new HostnameHandler()) ,DOMAINNAME(new DomainHandler()) + ,URI(new URIHandler()) ,; ObjectTypeHandler _handler; ObjectTypeHandlers(ObjectTypeHandler handler) { diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java new file mode 100644 index 0000000000..2adfb8a9ff --- /dev/null +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.dataloads.extractor.stix.types; + +import com.google.common.collect.ImmutableList; +import org.apache.metron.enrichment.converter.EnrichmentKey; +import org.apache.metron.enrichment.converter.EnrichmentValue; +import org.apache.metron.enrichment.lookup.LookupKV; +import org.mitre.cybox.common_2.AnyURIObjectPropertyType; +import org.mitre.cybox.objects.URIObjectType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class URIHandler extends AbstractObjectTypeHandler { + public URIHandler() { + super(URIObjectType.class); + } + + @Override + public Iterable extract(URIObjectType type, Map config) throws IOException { + List ret = new ArrayList<>(); + if(type != null) { + AnyURIObjectPropertyType val = type.getValue(); + if(val != null) { + Object v = val.getValue(); + if(v != null) { + final String indicatorType = getType(); + LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, v.toString()) + , new EnrichmentValue( + new HashMap() {{ + put("source-type", "STIX"); + put("uri", v.toString()); + put("indicator-type", indicatorType); + put("source", type.toXMLString()); + }} + ) + ); + ret.add(results); + } + } + } + return ret; + } + + @Override + public List getPossibleTypes() { + return ImmutableList.of(getType()); + } +} diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java new file mode 100644 index 0000000000..ff41152b23 --- /dev/null +++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.dataloads.extractor.stix; + +import com.google.common.collect.Iterables; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.dataloads.extractor.stix.types.URIHandler; +import org.apache.metron.enrichment.converter.EnrichmentKey; +import org.apache.metron.enrichment.lookup.LookupKV; +import org.junit.Assert; +import org.junit.Test; +import org.mitre.cybox.objects.URIObjectType; + +import java.util.HashMap; +import java.util.List; + +public class URIHandlerTest { + + /** + * + * + * + * + * + * ../../../../descendant-or-self::node() + * + * + * TBD + * + * + * Unclassified (Public) + * + * + * + * + * + * + * URL: http://www.kotimi.com/alpha/gtex/... + * URL: http://www.kotimi.com/alpha/gtex/| isOnline:yes| dateVerified:2017-07-31T22:03:10+00:00 + * + * + * http://www.kotimi.com/alpha/gtex/ + * + * + * + * + * + */ + @Multiline + static String uriHandlerObject; + + @Test + public void testURIHandler() throws Exception { + StixExtractor extractor = new StixExtractor(); + extractor.initialize(new HashMap<>()); + Iterable kvs = extractor.extract(uriHandlerObject); + Assert.assertEquals(1, Iterables.size(kvs)); + LookupKV kv = Iterables.getFirst(kvs, null); + EnrichmentKey key = (EnrichmentKey) kv.getKey(); + Assert.assertEquals("http://www.kotimi.com/alpha/gtex/", key.getIndicator()); + Assert.assertEquals("uriobjecttype", key.type); + } +} From faa6c07e66257b11573dce29b39495b8fbdfb87c Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 10 Aug 2017 11:07:46 -0400 Subject: [PATCH 2/4] Wrong logging framework, whoops --- .../dataloads/extractor/stix/StixExtractor.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java index c926b79f1a..f96bd2d74c 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java @@ -19,7 +19,6 @@ import com.google.common.base.Splitter; import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; import org.apache.metron.dataloads.extractor.Extractor; import org.apache.metron.dataloads.extractor.stix.types.ObjectTypeHandler; import org.apache.metron.dataloads.extractor.stix.types.ObjectTypeHandlers; @@ -31,15 +30,18 @@ import org.mitre.stix.common_1.IndicatorBaseType; import org.mitre.stix.indicator_2.Indicator; import org.mitre.stix.stix_1.STIXPackage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.Map; public class StixExtractor implements Extractor { - private static final Logger LOG = Logger.getLogger(StixExtractor.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); Map config; @Override public Iterable extract(String line) throws IOException { @@ -53,9 +55,9 @@ public Iterable extract(String line) throws IOException { ObjectTypeHandler handler = ObjectTypeHandlers.getHandlerByInstance(props); if (handler != null) { if(LOG.isDebugEnabled()) { - LOG.debug("Found " + handler.getTypeClass().getCanonicalName() - + " for properties " + props.toXMLString() - ); + LOG.debug("Found {} for properties {}" + , handler.getTypeClass().getCanonicalName() + , props.toXMLString()); } Iterable extractions = handler.extract(props, config); for(LookupKV extraction : extractions) { @@ -63,8 +65,9 @@ public Iterable extract(String line) throws IOException { } } else if(LOG.isDebugEnabled()) { - LOG.debug("Did not find a handler" - + " for properties " + props.toXMLString() + " of type " + props.getClass() + LOG.debug("Did not find a handler for properties {} of type {}" + , props.toXMLString() + , props.getClass() ); } } From e69e0b9c96dc377516e637532273420f770ad2bd Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 10 Aug 2017 11:12:58 -0400 Subject: [PATCH 3/4] Readme update --- metron-platform/metron-data-management/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/metron-platform/metron-data-management/README.md b/metron-platform/metron-data-management/README.md index 014f3efca1..cdb4155811 100644 --- a/metron-platform/metron-data-management/README.md +++ b/metron-platform/metron-data-management/README.md @@ -107,6 +107,7 @@ We support a subset of STIX messages for importation: | Address | MAC | address:MAC | | Domain | FQDN | domain:FQDN | | Hostname | | hostname | +| URI | | uriobjecttype | NOTE: The enrichment type will be used as the type above. From b2cc24f3e91edb5f096083c9617ba4171448c3d3 Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 10 Aug 2017 17:08:24 -0400 Subject: [PATCH 4/4] Updating documentation with support. --- metron-platform/metron-data-management/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metron-platform/metron-data-management/README.md b/metron-platform/metron-data-management/README.md index cdb4155811..c351f48e29 100644 --- a/metron-platform/metron-data-management/README.md +++ b/metron-platform/metron-data-management/README.md @@ -97,6 +97,11 @@ Consider the following config for importing STIX documents. This is a threat in format, so it is particularly relevant and attractive data to import for our purposes. Because STIX is a standard format, there is no need to specify the schema or how to interpret the documents. +We support the versions of Stix and Cybox supported by +[java-stix](https://github.com/STIXProject/java-stix/tree/v1.2.0.2): +* Stix - [1.2](https://github.com/STIXProject/schemas/blob/356cc4f6b06625465f0808388eb166807313b4e0/stix_core.xsd) and earlier +* Cybox - [2.1](https://github.com/CybOXProject/schemas/blob/97beb32c376a9223e91b52cb3e4c8d2af6baf786/cybox_core.xsd) and earlier + We support a subset of STIX messages for importation: | STIX Type | Specific Type | Enrichment Type Name |