>() {{
- add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
- }});
-
- BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03);
- PersistentAccessTracker pat = new PersistentAccessTracker(threatIntelTableName, "0", trackerTable, cf, bat, 0L);
- lookup = new EnrichmentLookup(threatIntelTable, cf, pat);
JSONParser jsonParser = new JSONParser();
expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
}
-
@Test
public void testEnrich() throws Exception {
- ThreatIntelAdapter tia = new ThreatIntelAdapter();
- tia.lookup = lookup;
SensorEnrichmentConfig broSc = JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
- JSONObject actualMessage = tia.enrich(new CacheKey("ip_dst_addr", "10.0.2.3", broSc));
+ JSONObject actualMessage = threatIntelAdapter.enrich(new CacheKey("ip_dst_addr", "10.0.2.3", broSc));
Assert.assertNotNull(actualMessage);
Assert.assertEquals(expectedMessage, actualMessage);
}
@Test
public void testEnrichNonString() throws Exception {
- ThreatIntelAdapter tia = new ThreatIntelAdapter();
- tia.lookup = lookup;
SensorEnrichmentConfig broSc = JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
- JSONObject actualMessage = tia.enrich(new CacheKey("ip_dst_addr", "10.0.2.3", broSc));
+ JSONObject actualMessage = threatIntelAdapter.enrich(new CacheKey("ip_dst_addr", "10.0.2.3", broSc));
Assert.assertNotNull(actualMessage);
Assert.assertEquals(expectedMessage, actualMessage);
- actualMessage = tia.enrich(new CacheKey("ip_dst_addr", 10L, broSc));
+ actualMessage = threatIntelAdapter.enrich(new CacheKey("ip_dst_addr", 10L, broSc));
Assert.assertEquals(actualMessage,new JSONObject());
}
+ @Test
+ public void testMiss() {
+ JSONObject actual = threatIntelAdapter.enrich(new CacheKey("ip_dst_addr", "4.4.4.4", config));
+
+ // not a known IP in either the enrichment data
+ JSONObject expected = new JSONObject();
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testMissWithNonStringValue() {
+ JSONObject actual = threatIntelAdapter.enrich(new CacheKey("ip_dst_addr", 10L, config));
+
+ // not a known IP in either the enrichment data
+ JSONObject expected = new JSONObject();
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testNoEnrichmentDefined() {
+ JSONObject actual = threatIntelAdapter.enrich(new CacheKey("username", "ada_lovelace", config));
+
+ // no enrichment defined for the field 'username'
+ JSONObject expected = new JSONObject();
+ Assert.assertEquals(expected, actual);
+ }
+
@Test
public void testInitializeAdapter() {
@@ -155,12 +169,14 @@ public void testInitializeAdapter() {
config.withMillisecondsBetweenPersists(millionseconds);
config.withTrackerHBaseCF(trackCf);
config.withTrackerHBaseTable(trackTable);
- config.withProviderImpl(ExceptionProvider.class.getName());
+ config.withEnrichmentLookupFactory(new FakeEnrichmentLookupFactory(lookup));
+ config.withConnectionFactory(new FakeHBaseConnectionFactory());
+
ThreatIntelAdapter tia = new ThreatIntelAdapter(config);
UnitTestHelper.setLog4jLevel(ThreatIntelAdapter.class, Level.FATAL);
tia.initializeAdapter(null);
UnitTestHelper.setLog4jLevel(ThreatIntelAdapter.class, Level.ERROR);
- Assert.assertFalse(tia.isInitialized());
+ Assert.assertTrue(tia.isInitialized());
}
diff --git a/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/EnrichmentLookupFactoriesTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/EnrichmentLookupFactoriesTest.java
new file mode 100644
index 0000000000..1b865eb010
--- /dev/null
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/EnrichmentLookupFactoriesTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.lookup;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+
+public class EnrichmentLookupFactoriesTest {
+
+ @Test
+ public void byEnumName() {
+ assertNotNull(EnrichmentLookupFactories.byName(EnrichmentLookupFactories.HBASE.name()));
+ }
+
+ @Test
+ public void byClassName() {
+ assertNotNull(EnrichmentLookupFactories.byName(FakeEnrichmentLookupFactory.class.getName()));
+ }
+
+ @Test(expected=IllegalStateException.class)
+ public void shouldFailWithInvalidName() {
+ EnrichmentLookupFactories.byName("this-is-an-invalid-name");
+ }
+}
diff --git a/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/FakeEnrichmentLookup.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/FakeEnrichmentLookup.java
new file mode 100644
index 0000000000..e99994d0a3
--- /dev/null
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/FakeEnrichmentLookup.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.lookup;
+
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.lookup.handler.KeyWithContext;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An {@link EnrichmentLookup} useful for testing.
+ *
+ * Maintains a static, in-memory set of enrichments to mimic the behavior of
+ * an {@link EnrichmentLookup} that interacts with HBase.
+ */
+public class FakeEnrichmentLookup extends EnrichmentLookup implements Serializable {
+
+ /**
+ * The available enrichments. This is static so that all
+ * instances 'see' the same set of enrichments.
+ */
+ private static Map enrichments = Collections.synchronizedMap(new HashMap<>());
+
+ public FakeEnrichmentLookup() {
+ super(null, null, null);
+ }
+
+ /**
+ * Add an enrichment.
+ * @param key The enrichment key.
+ * @param value The enrichment value.
+ * @return
+ */
+ public FakeEnrichmentLookup withEnrichment(EnrichmentKey key, EnrichmentValue value) {
+ this.enrichments.put(key, value);
+ return this;
+ }
+
+ /**
+ * Deletes all enrichments.
+ */
+ public FakeEnrichmentLookup deleteAll() {
+ enrichments.clear();
+ return this;
+ }
+
+ @Override
+ public boolean exists(EnrichmentKey key, HBaseContext context, boolean logAccess) {
+ return enrichments.containsKey(key);
+ }
+
+ @Override
+ public Iterable exists(Iterable> keys, boolean logAccess) throws IOException {
+ List results = new ArrayList<>();
+ for(KeyWithContext keyWithContext: keys) {
+ EnrichmentKey key = keyWithContext.getKey();
+ results.add(enrichments.containsKey(key));
+ }
+ return results;
+ }
+
+ @Override
+ public LookupKV get(EnrichmentKey key, HBaseContext context, boolean logAccess) {
+ EnrichmentValue value = enrichments.get(key);
+ return new LookupKV<>(key, value);
+ }
+
+ @Override
+ public Iterable> get(Iterable> keys, boolean logAccess) throws IOException {
+ List> results = new ArrayList<>();
+ for(KeyWithContext keyWithContext: keys) {
+ EnrichmentKey key = keyWithContext.getKey();
+ HBaseContext context = keyWithContext.getContext();
+ if(enrichments.containsKey(key)) {
+ results.add(get(key, context, logAccess));
+ }
+ }
+ return results;
+ }
+
+
+
+ @Override
+ public void close() throws IOException {
+ // nothing to do
+ }
+}
diff --git a/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/FakeEnrichmentLookupFactory.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/FakeEnrichmentLookupFactory.java
new file mode 100644
index 0000000000..6974a50af6
--- /dev/null
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/lookup/FakeEnrichmentLookupFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.lookup;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.metron.enrichment.lookup.accesstracker.AccessTracker;
+import org.apache.metron.hbase.client.HBaseConnectionFactory;
+
+/**
+ * Creates a {@link FakeEnrichmentLookup}.
+ */
+public class FakeEnrichmentLookupFactory implements EnrichmentLookupFactory {
+
+ private FakeEnrichmentLookup lookup;
+
+ public FakeEnrichmentLookupFactory() {
+ this(new FakeEnrichmentLookup());
+ }
+
+ public FakeEnrichmentLookupFactory(FakeEnrichmentLookup lookup) {
+ this.lookup = lookup;
+ }
+
+ @Override
+ public EnrichmentLookup create(HBaseConnectionFactory connectionFactory,
+ Configuration configuration,
+ String tableName,
+ String columnFamily,
+ AccessTracker accessTracker) {
+ return lookup;
+ }
+}
diff --git a/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
index dbbc7d5dbc..3ec08d764f 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
@@ -19,59 +19,84 @@
package org.apache.metron.enrichment.stellar;
import com.google.common.collect.ImmutableMap;
-import org.apache.metron.hbase.mock.MockHTable;
-import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.metron.enrichment.lookup.EnrichmentLookupFactory;
+import org.apache.metron.enrichment.lookup.FakeEnrichmentLookup;
+import org.apache.metron.hbase.client.FakeHBaseConnectionFactory;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.DefaultVariableResolver;
import org.apache.metron.stellar.dsl.ParseException;
-import org.apache.metron.stellar.dsl.StellarFunctions;
import org.apache.metron.stellar.common.StellarProcessor;
-import org.apache.metron.enrichment.converter.EnrichmentHelper;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.stellar.dsl.VariableResolver;
+import org.apache.metron.stellar.dsl.functions.FunctionalFunctions;
+import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
+import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static com.google.common.collect.ImmutableMap.of;
+import static org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions.EnrichmentExists;
+import static org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions.EnrichmentGet;
+
public class SimpleHBaseEnrichmentFunctionsTest {
- private final String hbaseTableName = "enrichments";
private static final String ENRICHMENT_TYPE = "et";
- private String cf = "cf";
private static Context context;
-
-
+ private EnrichmentExists existsFunction;
+ private EnrichmentGet getFunction;
@Before
public void setup() throws Exception {
+ // provides the enrichment data to the functions
+ FakeEnrichmentLookup lookup = new FakeEnrichmentLookup()
+ .withEnrichment(new EnrichmentKey(ENRICHMENT_TYPE, "indicator0"), new EnrichmentValue(of("key0", "value0")))
+ .withEnrichment(new EnrichmentKey(ENRICHMENT_TYPE, "indicator1"), new EnrichmentValue(of("key1", "value1")))
+ .withEnrichment(new EnrichmentKey(ENRICHMENT_TYPE, "indicator2"), new EnrichmentValue(of("key2", "value2")))
+ .withEnrichment(new EnrichmentKey(ENRICHMENT_TYPE, "indicator3"), new EnrichmentValue(of("key3", "value3")))
+ .withEnrichment(new EnrichmentKey(ENRICHMENT_TYPE, "indicator4"), new EnrichmentValue(of("key4", "value4")));
- final MockHTable hbaseTable = (MockHTable) MockHBaseTableProvider.addToCache(hbaseTableName, cf);
- EnrichmentHelper.INSTANCE.load(hbaseTable, cf, new ArrayList>() {{
- for(int i = 0;i < 5;++i) {
- add(new LookupKV<>(new EnrichmentKey(ENRICHMENT_TYPE, "indicator" + i)
- , new EnrichmentValue(ImmutableMap.of("key" + i, "value" + i))
- )
- );
- }
- }});
context = new Context.Builder()
.with( Context.Capabilities.GLOBAL_CONFIG
- , () -> ImmutableMap.of( SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF
- , MockHBaseTableProvider.class.getName()
+ , () -> ImmutableMap.of( SimpleHBaseEnrichmentFunctions.CONNECTION_FACTORY_IMPL_CONF
+ , FakeHBaseConnectionFactory.class.getName()
)
)
.build();
+
+ EnrichmentLookupFactory factory = (connFact, conf, tableName, colFam, accessTracker) -> lookup;
+ Configuration configuration = HBaseConfiguration.create();
+
+ // the ENRICHMENT_EXIST function to test
+ existsFunction = new EnrichmentExists(factory, configuration);
+ existsFunction.initialize(context);
+
+ // the ENRICHMENT_GET function to test
+ getFunction = new EnrichmentGet(factory, configuration);
+ getFunction.initialize(context);
}
- public Object run(String rule, Map variables) throws Exception {
+
+ public Object run(String rule, Map variables) {
StellarProcessor processor = new StellarProcessor();
Assert.assertTrue(rule + " not valid.", processor.validate(rule, context));
- return processor.parse(rule, new DefaultVariableResolver(x -> variables.get(x),x -> variables.containsKey(x)), StellarFunctions.FUNCTION_RESOLVER(), context);
+
+ VariableResolver variableResolver = new DefaultVariableResolver(
+ x -> variables.get(x),
+ x -> variables.containsKey(x));
+ FunctionResolver functionResolver = new SimpleFunctionResolver()
+ .withClass(EnrichmentGet.class)
+ .withClass(EnrichmentExists.class)
+ .withClass(FunctionalFunctions.Map.class)
+ .withInstance(existsFunction)
+ .withInstance(getFunction);
+ return processor.parse(rule, variableResolver, functionResolver, context);
}
@Test
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
index fe21399b89..7f1175e686 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
@@ -34,6 +34,19 @@
+
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ ${global_jackson_version}
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+ ${global_jackson_version}
+
+
@@ -202,16 +215,6 @@
caffeine
${global_caffeine_version}
-
- com.fasterxml.jackson.core
- jackson-databind
- ${global_jackson_version}
-
-
- com.fasterxml.jackson.core
- jackson-annotations
- ${global_jackson_version}
-
org.slf4j
slf4j-api
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment.properties
index 5338ead22c..f93405e066 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment.properties
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment.properties
@@ -41,12 +41,12 @@ enrichment.join.cache.size=100000
threat.intel.join.cache.size=100000
##### Enrichment #####
-hbase.provider.impl=org.apache.metron.hbase.HTableProvider
enrichment.simple.hbase.table=enrichment
enrichment.simple.hbase.cf=t
enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+enrichment.lookup.factory="HBASE"
##### Threat Intel #####
threat.intel.tracker.table=access_tracker
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment.properties.j2 b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment.properties.j2
index 8c28c49c48..491818d616 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment.properties.j2
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment.properties.j2
@@ -34,10 +34,10 @@ enrichment.error.topic={{enrichment_error_topic}}
threat.intel.error.topic={{threatintel_error_topic}}
##### Enrichment #####
-hbase.provider.impl={{enrichment_hbase_provider_impl}}
enrichment.simple.hbase.table={{enrichment_hbase_table}}
enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+enrichment.lookup.factory={{enrichment_lookup_factory}}
##### Threat Intel #####
threat.intel.tracker.table={{threatintel_hbase_table}}
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/flux/enrichment/remote.yaml
index 45b05cfdc6..9f46fa6e2d 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/flux/enrichment/remote.yaml
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/flux/enrichment/remote.yaml
@@ -78,15 +78,15 @@ components:
- id: "simpleHBaseEnrichmentConfig"
className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseConfig"
configMethods:
- - name: "withProviderImpl"
- args:
- - "${hbase.provider.impl}"
- name: "withHBaseTable"
args:
- - "${enrichment.simple.hbase.table}"
+ - "${enrichment.simple.hbase.table}"
- name: "withHBaseCF"
args:
- - "${enrichment.simple.hbase.cf}"
+ - "${enrichment.simple.hbase.cf}"
+ - name: "withEnrichmentLookupFactory"
+ args:
+ - "${enrichment.lookup.factory}"
- id: "simpleHBaseEnrichmentAdapter"
className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter"
@@ -148,21 +148,21 @@ components:
- id: "simpleHBaseThreatIntelConfig"
className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelConfig"
configMethods:
- - name: "withProviderImpl"
- args:
- - "${hbase.provider.impl}"
- name: "withTrackerHBaseTable"
args:
- - "${threat.intel.tracker.table}"
+ - "${threat.intel.tracker.table}"
- name: "withTrackerHBaseCF"
args:
- - "${threat.intel.tracker.cf}"
+ - "${threat.intel.tracker.cf}"
- name: "withHBaseTable"
args:
- - "${threat.intel.simple.hbase.table}"
+ - "${threat.intel.simple.hbase.table}"
- name: "withHBaseCF"
args:
- - "${threat.intel.simple.hbase.cf}"
+ - "${threat.intel.simple.hbase.cf}"
+ - name: "withEnrichmentLookupFactory"
+ args:
+ - "${enrichment.lookup.factory}"
- id: "simpleHBaseThreatIntelAdapter"
className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelAdapter"
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index a138c0d897..105b4a02d1 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -24,20 +24,21 @@
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.metron.TestConstants;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.enrichment.adapters.maxmind.asn.GeoLiteAsnDatabase;
import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase;
-import org.apache.metron.enrichment.converter.EnrichmentHelper;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.lookup.LookupKV;
-import org.apache.metron.enrichment.lookup.accesstracker.PersistentBloomTrackerCreator;
+import org.apache.metron.enrichment.lookup.EnrichmentLookupFactory;
+import org.apache.metron.enrichment.lookup.FakeEnrichmentLookup;
+import org.apache.metron.enrichment.lookup.FakeEnrichmentLookupFactory;
+import org.apache.metron.enrichment.lookup.accesstracker.AccessTrackers;
import org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions;
import org.apache.metron.enrichment.utils.ThreatIntelUtils;
-import org.apache.metron.hbase.mock.MockHBaseTableProvider;
-import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.integration.BaseIntegrationTest;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.ProcessorResult;
@@ -48,6 +49,7 @@
import org.apache.metron.integration.processors.KafkaMessageSet;
import org.apache.metron.integration.processors.KafkaProcessor;
import org.apache.metron.integration.utils.TestUtils;
+import org.apache.metron.stellar.dsl.StellarFunctions;
import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.parser.ParseException;
import org.junit.Assert;
@@ -67,6 +69,9 @@
import java.util.Set;
import java.util.stream.Stream;
+import static org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions.EnrichmentExists;
+import static org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions.EnrichmentGet;
+
/**
* Integration test for the enrichment topology.
*/
@@ -88,7 +93,6 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
public static final String DEFAULT_DMACODE= "test dmaCode";
public static final String DEFAULT_LOCATION_POINT= Joiner.on(',').join(DEFAULT_LATITUDE,DEFAULT_LONGITUDE);
public static final String cf = "cf";
- public static final String trackerHBaseTableName = "tracker";
public static final String threatIntelTableName = "threat_intel";
public static final String enrichmentsTableName = "enrichments";
@@ -158,7 +162,7 @@ public Properties getTopologyProperties() {
setProperty("threatintel_error_topic", ERROR_TOPIC);
// enrichment
- setProperty("enrichment_hbase_provider_impl", "" + MockHBaseTableProvider.class.getName());
+ setProperty("enrichment_lookup_factory", FakeEnrichmentLookupFactory.class.getName());
setProperty("enrichment_hbase_table", enrichmentsTableName);
setProperty("enrichment_hbase_cf", cf);
setProperty("enrichment_host_known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"}," +
@@ -201,10 +205,7 @@ public void test() throws Exception {
{
File globalConfig = new File(enrichmentConfigPath, "global.json");
Map config = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER);
- config.put(SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF, MockHBaseTableProvider.class.getName());
- config.put(SimpleHBaseEnrichmentFunctions.ACCESS_TRACKER_TYPE_CONF, "PERSISTENT_BLOOM");
- config.put(PersistentBloomTrackerCreator.Config.PERSISTENT_BLOOM_TABLE, trackerHBaseTableName);
- config.put(PersistentBloomTrackerCreator.Config.PERSISTENT_BLOOM_CF, cf);
+ config.put(SimpleHBaseEnrichmentFunctions.ACCESS_TRACKER_TYPE_CONF, AccessTrackers.NOOP);
config.put(GeoLiteCityDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
config.put(GeoLiteAsnDatabase.ASN_HDFS_FILE, asnHdfsFile.getAbsolutePath());
globalConfigStr = JSONUtils.INSTANCE.toJSON(config, true);
@@ -214,19 +215,21 @@ public void test() throws Exception {
.withGlobalConfig(globalConfigStr)
.withEnrichmentConfigsPath(enrichmentConfigPath);
- //create MockHBaseTables
- final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(trackerHBaseTableName, cf);
- final MockHTable threatIntelTable = (MockHTable) MockHBaseTableProvider.addToCache(threatIntelTableName, cf);
- EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList>() {{
- add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
- }});
- final MockHTable enrichmentTable = (MockHTable) MockHBaseTableProvider.addToCache(enrichmentsTableName, cf);
- EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList>() {{
- add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3")
- , new EnrichmentValue(PLAYFUL_ENRICHMENT)
- )
- );
- }});
+ // add some enrichments to a set of global, static enrichment values to use during these tests
+ FakeEnrichmentLookup lookup = new FakeEnrichmentLookup()
+ .withEnrichment(
+ new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"),
+ new EnrichmentValue(new HashMap<>()))
+ .withEnrichment(
+ new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3"),
+ new EnrichmentValue(PLAYFUL_ENRICHMENT));
+ EnrichmentLookupFactory lookupCreator = (v, w, x, y, z) -> lookup;
+
+ // the enrichment stellar functions need to access the same global, static enrichment values
+ Configuration conf = HBaseConfiguration.create();
+ StellarFunctions.FUNCTION_RESOLVER()
+ .withInstance(new EnrichmentGet(lookupCreator, conf))
+ .withInstance(new EnrichmentExists(lookupCreator, conf));
FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
.withTopologyLocation(new File(fluxPath()))
@@ -235,7 +238,6 @@ public void test() throws Exception {
.withTopologyProperties(topologyProperties)
.build();
-
//UnitTestHelper.verboseLogging();
ComponentRunner runner = new ComponentRunner.Builder()
.withComponent("zk",zkServerComponent)
diff --git a/metron-platform/metron-hbase/metron-hbase-common/pom.xml b/metron-platform/metron-hbase/metron-hbase-common/pom.xml
index 84b24d3169..553b790fbc 100644
--- a/metron-platform/metron-hbase/metron-hbase-common/pom.xml
+++ b/metron-platform/metron-hbase/metron-hbase-common/pom.xml
@@ -52,6 +52,10 @@
org.slf4j
slf4j-log4j12
+
+ com.google.guava
+ guava
+
@@ -89,6 +93,10 @@
org.slf4j
slf4j-log4j12
+
+ com.google.guava
+ guava
+
@@ -97,6 +105,12 @@
tests
${global_hadoop_version}
provided
+
+
+ com.google.guava
+ guava
+
+
org.apache.hbase