diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index b1a2baced8..616f9cbe34 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -49,7 +49,7 @@ org.jruby.joni:joni:jar:2.1.2:compile,MIT License,https://github.com/jruby/joni org.mitre.taxii:taxii:jar:1.1.0.1:compile,The BSD 3-Clause License,https://github.com/TAXIIProject/java-taxii org.mitre:stix:jar:1.2.0.2:compile,The BSD 3-Clause License,https://github.com/STIXProject/java-stix org.mockito:mockito-core:jar:1.10.19:compile,The MIT License,http://www.mockito.org -org.scala-lang:scala-library:jar:2.9.2:compile,BSD-like,http://www.scala-lang.org/ +org.scala-lang:scala-library:jar:2.10.6:compile,BSD-like,http://www.scala-lang.org/ xmlenc:xmlenc:jar:0.52:compile,The BSD License,http://xmlenc.sourceforge.net asm:asm:jar:3.1:compile,BSD,http://asm.ow2.org/ com.sun.jersey.contribs:jersey-guice:jar:1.9:compile,CDDL 1.1,https://jersey.java.net/ diff --git a/metron-analytics/metron-profiler-common/src/main/resources/META-INF/LICENSE b/metron-analytics/metron-profiler-common/src/main/resources/META-INF/LICENSE index e5616e782d..cc1764e362 100644 --- a/metron-analytics/metron-profiler-common/src/main/resources/META-INF/LICENSE +++ b/metron-analytics/metron-profiler-common/src/main/resources/META-INF/LICENSE @@ -228,7 +228,7 @@ This product bundles leveldbjni-all 1.8, which is available under a "BSD Softwar This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni This product bundles asm 4.0, which is available under a "BSD Software License" license. For details, see http://asm.ow2.org/ -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java index d59f38d96a..ccbb09cea5 100644 --- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java +++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java @@ -20,7 +20,7 @@ package org.apache.metron.profiler.hbase; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.ProfilePeriod; import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder; diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index 6f12d16199..e67852970e 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -84,7 +84,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} @@ -93,6 +93,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + org.mockito mockito-all diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml index db9739281c..b1de56528f 100644 --- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml +++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml @@ -50,7 +50,7 @@ components: ref: "defaultExecutor" - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" @@ -73,7 +73,7 @@ components: spouts: - id: "kafkaSpout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "kafkaConfig" diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java index 15c803ca6e..fa732e2b5f 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java @@ -20,14 +20,14 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.Config; -import backtype.storm.Constants; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.Config; +import org.apache.storm.Constants; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.metron.common.bolt.ConfiguredProfilerBolt; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.dsl.Context; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java index d6377e4f39..28d0530bc0 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java @@ -20,7 +20,7 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.commons.beanutils.BeanMap; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.dsl.ParseException; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java index 5d97e4d8bf..c784c8e27d 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java @@ -20,12 +20,12 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.commons.lang.StringUtils; import org.apache.metron.common.bolt.ConfiguredProfilerBolt; import org.apache.metron.common.configuration.profiler.ProfileConfig; diff --git a/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE b/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE index e5616e782d..cc1764e362 100644 --- a/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE +++ b/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE @@ -228,7 +228,7 @@ This product bundles leveldbjni-all 1.8, which is available under a "BSD Softwar This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni This product bundles asm 4.0, which is available under a "BSD Software License" license. For details, see http://asm.ow2.org/ -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java index caeddee7e7..48ff663372 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java @@ -20,9 +20,9 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.Constants; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.Constants; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.utils.JSONUtils; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java index 97caff95e3..f2fa2f63a4 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java @@ -20,7 +20,7 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.hbase.RowKeyBuilder; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java index 31ff279837..0ac1c332ec 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java @@ -20,8 +20,8 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.profiler.stellar.DefaultStellarExecutor; import org.apache.metron.test.bolt.BaseBoltTest; diff --git a/metron-deployment/roles/ambari_config/defaults/main.yml b/metron-deployment/roles/ambari_config/defaults/main.yml index 0badef1f89..9ac0342c46 100644 --- a/metron-deployment/roles/ambari_config/defaults/main.yml +++ b/metron-deployment/roles/ambari_config/defaults/main.yml @@ -34,4 +34,4 @@ mapred_reduce_java_opts : -Xmx1024m mapred_map_mem_mb : 1229 mapred_reduce_mem_mb : 1229 topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf' -hdp_stack: "2.4" +hdp_stack: "2.5" diff --git a/metron-deployment/roles/flume/defaults/main.yml b/metron-deployment/roles/flume/defaults/main.yml index 38ed75521f..d96c844cac 100644 --- a/metron-deployment/roles/flume/defaults/main.yml +++ b/metron-deployment/roles/flume/defaults/main.yml @@ -15,5 +15,5 @@ # limitations under the License. # --- -hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.2.0/hdp.repo" +hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo" java_home: /usr/jdk64/jdk1.8.0_60 diff --git a/metron-deployment/roles/kafka-broker/defaults/main.yml b/metron-deployment/roles/kafka-broker/defaults/main.yml index 8e22e9fd0a..b0f5a11400 100644 --- a/metron-deployment/roles/kafka-broker/defaults/main.yml +++ b/metron-deployment/roles/kafka-broker/defaults/main.yml @@ -15,4 +15,4 @@ # limitations under the License. # --- -hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.2.0/hdp.repo" +hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo" diff --git a/metron-deployment/roles/kafka-client/defaults/main.yml b/metron-deployment/roles/kafka-client/defaults/main.yml index 8e22e9fd0a..b0f5a11400 100644 --- a/metron-deployment/roles/kafka-client/defaults/main.yml +++ b/metron-deployment/roles/kafka-client/defaults/main.yml @@ -15,4 +15,4 @@ # limitations under the License. # --- -hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.2.0/hdp.repo" +hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo" diff --git a/metron-platform/metron-api/pom.xml b/metron-platform/metron-api/pom.xml index d600bfddc0..3c9c9e5a64 100644 --- a/metron-platform/metron-api/pom.xml +++ b/metron-platform/metron-api/pom.xml @@ -29,7 +29,7 @@ 3.4.5.2.0.6.0-76 1.2.15 - 0.9.2-incubating + 1.0.1 0.0.7-SNAPSHOT 0.0.5-SNAPSHOT diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index c19a507cd3..9faa256b97 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -86,7 +86,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} provided diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java index 9318bf0b86..cef4affa4a 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java @@ -17,9 +17,9 @@ */ package org.apache.metron.common.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.base.BaseRichBolt; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.base.BaseRichBolt; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java index 8f0a6afbfe..c26d1f5632 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java @@ -18,9 +18,9 @@ package org.apache.metron.common.spout.kafka; -import storm.kafka.BrokerHosts; +import org.apache.storm.kafka.BrokerHosts; -public class SpoutConfig extends storm.kafka.SpoutConfig { +public class SpoutConfig extends org.apache.storm.kafka.SpoutConfig { public static enum Offset { BEGINNING, END, WHERE_I_LEFT_OFF; } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java index e8abd60756..5aec998a62 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java @@ -19,6 +19,6 @@ package org.apache.metron.common.spout.kafka; public interface SpoutConfigFunction { - void configure(storm.kafka.SpoutConfig config, Object val); + void configure(org.apache.storm.kafka.SpoutConfig config, Object val); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java index 807694e15b..7d1c7c331f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java @@ -20,7 +20,7 @@ import com.google.common.base.Joiner; import org.apache.metron.common.utils.ConversionUtils; -import storm.kafka.SpoutConfig; +import org.apache.storm.kafka.SpoutConfig; import java.util.EnumMap; import java.util.Map; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java index 92e0ab91cc..3b3f42683f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java @@ -24,8 +24,8 @@ import java.util.List; import java.util.Optional; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Values; import org.apache.commons.beanutils.Converter; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.ArrayUtils; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java index 0e1aa90525..64e16b1b3e 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java @@ -17,7 +17,9 @@ */ package org.apache.metron.common.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; +import org.apache.metron.common.configuration.Configurations; +import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.writer.WriterConfiguration; import java.io.Serializable; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java index 2992358578..c4fa881d3d 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java @@ -18,7 +18,7 @@ package org.apache.metron.common.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java index 74339e95da..1ae54b15be 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.common.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.Configurations; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java index 64fdace475..fcd00fe65b 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.common.bolt; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Tuple; import org.apache.curator.test.TestingServer; import org.apache.metron.common.Constants; import org.apache.metron.TestConstants; diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java index a1bbc138af..ea864b9359 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.common.bolt; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Tuple; import org.apache.curator.test.TestingServer; import org.apache.metron.TestConstants; import org.apache.metron.common.Constants; diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml index 0263396ef0..f39139dc23 100644 --- a/metron-platform/metron-data-management/pom.xml +++ b/metron-platform/metron-data-management/pom.xml @@ -208,6 +208,14 @@ org.apache.hadoop hadoop-mapreduce-client-common + + log4j + log4j + + + log4j-over-slf4j + org.slf4j + diff --git a/metron-platform/metron-data-management/src/main/resources/META-INF/LICENSE b/metron-platform/metron-data-management/src/main/resources/META-INF/LICENSE index 19827c3550..52f4ee084c 100644 --- a/metron-platform/metron-data-management/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-data-management/src/main/resources/META-INF/LICENSE @@ -221,6 +221,6 @@ This product bundles jcodings 1.0.8, which is available under a "MIT Software Li This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni This product bundles taxii 1.1.0.1, which is available under a "BSD Software License" license. For details, see https://github.com/TAXIIProject/java-taxii This product bundles stix 1.2.0.2, which is available under a "BSD Software License" license. For details, see https://github.com/STIXProject/java-stix -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.5, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.5, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 4a3ce7afe3..6bb0979a8e 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -106,7 +106,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} @@ -115,6 +115,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + io.netty netty diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java index d8f34a9370..7360649b1a 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.elasticsearch.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; diff --git a/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE b/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE index 57abbc5d18..b3013a0668 100644 --- a/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE @@ -219,6 +219,6 @@ This product bundles jopt-simple 3.2, which is available under a "MIT Software L This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.7, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java index 51e7e0a2df..9aff560064 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java @@ -18,7 +18,7 @@ package org.apache.metron.elasticsearch.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.ImmutableList; import org.apache.metron.common.writer.BulkWriterResponse; import org.elasticsearch.action.bulk.BulkItemResponse; diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index b2c3a8c797..773c6b303d 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -121,7 +121,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} @@ -130,6 +130,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + org.apache.hadoop hadoop-common diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml index e182adebdb..03f48a0803 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml @@ -173,11 +173,11 @@ components: #kafka/zookeeper - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" - id: "kafkaConfig" - className: "storm.kafka.SpoutConfig" + className: "org.apache.storm.kafka.SpoutConfig" constructorArgs: # zookeeper hosts - ref: "zkHosts" @@ -195,7 +195,7 @@ components: spouts: - id: "kafkaSpout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "kafkaConfig" bolts: diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml index f8d7b83139..ca5868f46f 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml @@ -153,11 +153,11 @@ components: #kafka/zookeeper - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" - id: "kafkaConfig" - className: "storm.kafka.SpoutConfig" + className: "org.apache.storm.kafka.SpoutConfig" constructorArgs: # zookeeper hosts - ref: "zkHosts" @@ -175,7 +175,7 @@ components: spouts: - id: "kafkaSpout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "kafkaConfig" bolts: diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java index de6967bcfd..2adf4301a9 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java @@ -17,7 +17,7 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; +import org.apache.storm.task.TopologyContext; import com.google.common.base.Joiner; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java index af59dcd50c..1ec4252b8a 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java @@ -17,9 +17,9 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java index 059817a623..51d98b9593 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java @@ -18,12 +18,12 @@ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java index e0c96a2976..b31418c334 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java @@ -17,12 +17,12 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import com.google.common.base.Joiner; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java index e4b34a0c08..953e6a6dff 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java @@ -17,12 +17,12 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.metron.common.bolt.ConfiguredBolt; import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java index 01e5a9fbaf..fc92bff447 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java @@ -17,7 +17,7 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; +import org.apache.storm.task.TopologyContext; import com.google.common.base.Joiner; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java index 0314354167..13d88d5b6b 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java @@ -18,7 +18,7 @@ package org.apache.metron.enrichment.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import org.apache.hadoop.conf.Configuration; diff --git a/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE b/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE index 4858b45ed1..5819b8c5ce 100644 --- a/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE @@ -219,5 +219,5 @@ This product bundles jopt-simple 3.2, which is available under a "MIT Software L This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java index a692db4ed3..c53e053e0e 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java index 3b4fed0562..8fd3e73ea4 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java @@ -17,7 +17,7 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.tuple.Values; +import org.apache.storm.tuple.Values; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.TestConstants; diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java index 931297d808..5b06d336fa 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; -import backtype.storm.tuple.Values; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.test.bolt.BaseEnrichmentBoltTest; import org.json.simple.JSONObject; diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java index 6653b6d136..d0ca475156 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java @@ -17,10 +17,10 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import com.google.common.collect.ImmutableList; import junit.framework.Assert; import org.apache.metron.common.configuration.ConfigurationType; diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableConnector.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableConnector.java index ace4d8095e..504826e0dc 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableConnector.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableConnector.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.log4j.Logger; -import backtype.storm.generated.Bolt; +import org.apache.storm.generated.Bolt; /** * HTable connector for Storm {@link Bolt} diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TupleTableConfig.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TupleTableConfig.java index 8257d8aca1..e491184551 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TupleTableConfig.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TupleTableConfig.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.log4j.Logger; /** diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/BatchHelper.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/BatchHelper.java index e7cf2cfd50..9e3bbe0363 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/BatchHelper.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/BatchHelper.java @@ -20,9 +20,9 @@ package org.apache.metron.hbase.bolt; -import backtype.storm.Constants; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Tuple; +import org.apache.storm.Constants; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/HBaseBolt.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/HBaseBolt.java index ede1d8c9a5..6fc6b0af57 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/HBaseBolt.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/HBaseBolt.java @@ -20,12 +20,12 @@ package org.apache.metron.hbase.bolt; -import backtype.storm.Config; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichBolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.Config; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichBolt; +import org.apache.storm.tuple.Tuple; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Durability; diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/mapper/HBaseMapper.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/mapper/HBaseMapper.java index 6cae1b4fbb..6c7e5fbf3b 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/mapper/HBaseMapper.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/mapper/HBaseMapper.java @@ -20,7 +20,7 @@ package org.apache.metron.hbase.bolt.mapper; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import java.io.Serializable; import java.util.Optional; diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java index e53d38966c..f46cfde9c0 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.hbase.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTableInterface; diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/WidgetMapper.java b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/WidgetMapper.java index 4f97a5b599..6d56a4b405 100644 --- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/WidgetMapper.java +++ b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/WidgetMapper.java @@ -20,7 +20,7 @@ package org.apache.metron.hbase; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.hbase.bolt.mapper.ColumnList; import org.apache.metron.hbase.bolt.mapper.HBaseMapper; diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java index 621720e32e..f445b1ba44 100644 --- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java +++ b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java @@ -20,8 +20,8 @@ package org.apache.metron.hbase.bolt; -import backtype.storm.Constants; -import backtype.storm.tuple.Tuple; +import org.apache.storm.Constants; +import org.apache.storm.tuple.Tuple; import org.apache.metron.hbase.Widget; import org.apache.metron.hbase.WidgetMapper; import org.apache.metron.hbase.client.HBaseClient; diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java index 00a11266e3..27544c009d 100644 --- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java +++ b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java @@ -20,7 +20,7 @@ package org.apache.metron.hbase.client; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml index d1e697bb4c..8bf8f48a40 100644 --- a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml +++ b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml @@ -64,7 +64,7 @@ components: #kafka/zookeeper - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" - id: "kafkaConfig" @@ -85,7 +85,7 @@ components: spouts: - id: "kafkaSpout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "kafkaConfig" bolts: diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 14fe5c4863..2dbd87e147 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -76,7 +76,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} test @@ -86,17 +86,6 @@ - - org.apache.kafka - kafka_2.9.2 - ${global_kafka_version} - - - log4j - log4j - - - org.apache.hadoop @@ -148,6 +137,30 @@ + + + net.alchim31.maven + scala-maven-plugin + 3.2.1 + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java index 75be2d7d10..d48a981118 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java @@ -17,17 +17,17 @@ */ package org.apache.metron.integration.components; -import backtype.storm.Config; -import backtype.storm.LocalCluster; -import backtype.storm.generated.StormTopology; -import backtype.storm.generated.TopologyInfo; +import org.apache.storm.Config; +import org.apache.storm.LocalCluster; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.generated.TopologyInfo; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; import org.apache.storm.flux.FluxBuilder; import org.apache.storm.flux.model.ExecutionContext; import org.apache.storm.flux.model.TopologyDef; import org.apache.storm.flux.parser.FluxParser; -import org.apache.thrift7.TException; +import org.apache.storm.thrift.TException; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,11 +115,11 @@ public void stop() { } } - public void submitTopology() throws NoSuchMethodException, IOException, InstantiationException, TException, IllegalAccessException, InvocationTargetException, ClassNotFoundException { + public void submitTopology() throws NoSuchMethodException, IOException, InstantiationException, TException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, NoSuchFieldException { startTopology(getTopologyName(), getTopologyLocation(), getTopologyProperties()); } - private void startTopology(String topologyName, File topologyLoc, Properties properties) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, TException { + private void startTopology(String topologyName, File topologyLoc, Properties properties) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, TException, NoSuchFieldException{ TopologyDef topologyDef = loadYaml(topologyName, topologyLoc, properties); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java index 8ffc5e7a96..20ce721cf6 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java @@ -20,6 +20,7 @@ import com.google.common.base.Function; import kafka.admin.AdminUtils; +import kafka.admin.RackAwareMode; import kafka.api.FetchRequest; import kafka.api.FetchRequestBuilder; import kafka.common.TopicExistsException; @@ -38,7 +39,15 @@ import kafka.zk.EmbeddedZookeeper; import org.I0Itec.zkclient.ZkClient; import org.apache.metron.integration.InMemoryComponent; +import org.apache.metron.integration.wrapper.AdminUtilsWrapper; +import org.apache.metron.integration.wrapper.TestUtilsWrapper; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.*; @@ -61,6 +70,9 @@ public Topic(String name, int numPartitions) { private transient ZkClient zkClient; private transient ConsumerConnector consumer; private String zookeeperConnectString; +// def zkPort: Int = zookeeper.port +// def zkConnect: String = s + private int brokerPort = 6667; private List topics = Collections.emptyList(); private Function postStartCallback; @@ -78,8 +90,9 @@ public KafkaWithZKComponent withExistingZookeeper(String zookeeperConnectString) public KafkaWithZKComponent withBrokerPort(int brokerPort) { if(brokerPort <= 0) { - brokerPort = TestUtils.choosePort(); + brokerPort = TestUtils.RandomPort(); } + this.brokerPort = brokerPort; return this; } @@ -129,14 +142,17 @@ public KafkaProducer createProducer(Map properties, C public void start() { // setup Zookeeper if(zookeeperConnectString == null) { - String zkConnect = TestZKUtils.zookeeperConnect(); - zkServer = new EmbeddedZookeeper(zkConnect); - zookeeperConnectString = zkServer.connectString(); +// String zkConnect = TestZKUtils.zookeeperConnect(); +// zkServer = new EmbeddedZookeeper(zkConnect); +// zookeeperConnectString = zkServer.connectString(); + EmbeddedZookeeper ezk = new EmbeddedZookeeper(); + zookeeperConnectString = "127.0.0.1:" + ezk.port(); } zkClient = new ZkClient(zookeeperConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); // setup Broker - Properties props = TestUtils.createBrokerConfig(0, brokerPort, true); +// Properties props = TestUtilsWrapper.createBrokerConfig(0, brokerPort, true); + Properties props = TestUtilsWrapper.createBrokerConfig(0, zookeeperConnectString, brokerPort); props.setProperty("zookeeper.connection.timeout.ms","1000000"); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); @@ -192,7 +208,8 @@ public ConsumerIterator getStreamIterator(String topic) { } public ConsumerIterator getStreamIterator(String topic, String group, String consumerName) { // setup simple consumer - Properties consumerProperties = TestUtils.createConsumerProperties(zkServer.connectString(), group, consumerName, -1); +// Properties consumerProperties = TestUtils.createConsumerProperties(zkServer.connectString(), group, consumerName, -1); + Properties consumerProperties = TestUtils.createConsumerProperties(zookeeperConnectString, group, consumerName, -1); consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(consumerProperties)); Map topicCountMap = new HashMap(); topicCountMap.put(topic, 1); @@ -220,7 +237,9 @@ public void waitUntilMetadataIsPropagated(String topic, int numPartitions) { public void createTopic(String name, int numPartitions, boolean waitUntilMetadataIsPropagated) throws InterruptedException { try { - AdminUtils.createTopic(zkClient, name, numPartitions, 1, new Properties()); + ZkUtils zkUtils = ZkUtils.apply(zookeeperConnectString, 30000, 30000, false); + AdminUtilsWrapper.createTopic(zkUtils, name, numPartitions, 1, new Properties()); +// AdminUtils.createTopic(zkClient, name, numPartitions, 1, new Properties()); if (waitUntilMetadataIsPropagated) { waitUntilMetadataIsPropagated(name, numPartitions); } diff --git a/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/AdminUtilsWrapper.scala b/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/AdminUtilsWrapper.scala new file mode 100644 index 0000000000..aed9dda0f3 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/AdminUtilsWrapper.scala @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.integration.wrapper + +import java.util.Properties + +import kafka.admin.AdminUtils +import kafka.utils.ZkUtils + +object AdminUtilsWrapper { + def createTopic(zkUtils: ZkUtils, + topic: String, + partitions: Int, + replicationFactor: Int, + topicConfig: Properties = new Properties) { + AdminUtils.createTopic(zkUtils, topic, partitions, replicationFactor, topicConfig) + } +} diff --git a/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/TestUtilsWrapper.scala b/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/TestUtilsWrapper.scala new file mode 100644 index 0000000000..576c8a2bd2 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/TestUtilsWrapper.scala @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.integration.wrapper + +import java.util.Properties + +import kafka.utils.TestUtils + +object TestUtilsWrapper { + // Don't expose properties with defaults + def createBrokerConfig(nodeId: Int, zkConnect: String, port: Int): Properties = { + TestUtils.createBrokerConfig(nodeId = nodeId, zkConnect = zkConnect, port = port) + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index 25fbe8b728..3e42b35c46 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -139,7 +139,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} @@ -148,6 +148,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + commons-cli commons-cli diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 325209f880..476df5bc47 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -17,12 +17,12 @@ */ package org.apache.metron.parsers.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredParserBolt; import org.apache.metron.common.configuration.FieldValidator; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java index 380f760b7f..8eb065614d 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java @@ -18,11 +18,11 @@ package org.apache.metron.parsers.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichBolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichBolt; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.utils.ErrorUtils; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java index 24d803f94e..38425b5891 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java @@ -18,8 +18,8 @@ package org.apache.metron.parsers.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java index d5ec0558f7..5f1927e9bc 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java @@ -17,7 +17,7 @@ */ package org.apache.metron.parsers.topology; -import backtype.storm.topology.TopologyBuilder; +import org.apache.storm.topology.TopologyBuilder; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.ConfigurationsUtils; @@ -36,8 +36,8 @@ import org.apache.metron.writer.AbstractWriter; import org.apache.metron.writer.kafka.KafkaWriter; import org.json.simple.JSONObject; -import storm.kafka.KafkaSpout; -import storm.kafka.ZkHosts; +import org.apache.storm.kafka.KafkaSpout; +import org.apache.storm.kafka.ZkHosts; import java.util.EnumMap; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java index 0ebec6c1c7..5ea561c709 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java @@ -17,12 +17,11 @@ */ package org.apache.metron.parsers.topology; -import backtype.storm.Config; -import backtype.storm.ConfigValidation; -import backtype.storm.LocalCluster; -import backtype.storm.StormSubmitter; -import backtype.storm.topology.TopologyBuilder; -import backtype.storm.utils.Utils; +import org.apache.storm.Config; +import org.apache.storm.LocalCluster; +import org.apache.storm.StormSubmitter; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.utils.Utils; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Joiner; import org.apache.commons.cli.*; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java index add11b7d65..c746a56f93 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java @@ -18,7 +18,7 @@ package org.apache.metron.parsers.topology.config; -import backtype.storm.Config; +import org.apache.storm.Config; public class Arg { private Config config; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java index f982e29489..f6c04a6c5d 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java @@ -18,7 +18,7 @@ package org.apache.metron.parsers.topology.config; -import backtype.storm.Config; +import org.apache.storm.Config; import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.io.FileUtils; import org.apache.metron.common.utils.JSONUtils; diff --git a/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE b/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE index 7b3f8d0b02..0970a173b2 100644 --- a/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE @@ -219,7 +219,7 @@ This product bundles jopt-simple 3.2, which is available under a "MIT Software L This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index 5d938384fc..cdebbc3fd2 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -19,8 +19,8 @@ import org.apache.metron.common.configuration.SensorParserConfig; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.ImmutableList; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java index 083a4e3c6c..e8e2653d77 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java @@ -17,9 +17,11 @@ */ package org.apache.metron.parsers.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.tuple.Tuple; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java index b3d09d07d5..14fe748690 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java @@ -62,7 +62,7 @@ public void test() throws Exception { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("kafka", kafkaComponent) .withComponent("config", configUploadComponent) - .withComponent("storm", parserTopologyComponent) + .withComponent("org/apache/storm", parserTopologyComponent) .withMillisecondsBetweenAttempts(5000) .withNumRetries(10) .build(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java index f639e0269e..80b6ebdb87 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java @@ -17,9 +17,9 @@ */ package org.apache.metron.parsers.integration.components; -import backtype.storm.Config; -import backtype.storm.LocalCluster; -import backtype.storm.topology.TopologyBuilder; +import org.apache.storm.Config; +import org.apache.storm.LocalCluster; +import org.apache.storm.topology.TopologyBuilder; import org.apache.metron.common.spout.kafka.SpoutConfig; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java index 2940bfe416..83a0b6acca 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java @@ -18,7 +18,7 @@ package org.apache.metron.parsers.topology; -import backtype.storm.Config; +import org.apache.storm.Config; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.cli.CommandLine; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java index 2f359ef521..6dab0c5c46 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java @@ -96,7 +96,7 @@ public void test() throws UnableToStartException, IOException { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("kafka", kafkaComponent) .withComponent("config", configUploadComponent) - .withComponent("storm", parserTopologyComponent) + .withComponent("org/apache/storm", parserTopologyComponent) .withMillisecondsBetweenAttempts(5000) .withNumRetries(10) .build(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java index b9377f3ff2..f97619fcef 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java @@ -112,7 +112,7 @@ public void test() throws UnableToStartException, IOException { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("kafka", kafkaComponent) .withComponent("config", configUploadComponent) - .withComponent("storm", parserTopologyComponent) + .withComponent("org/apache/storm", parserTopologyComponent) .withMillisecondsBetweenAttempts(5000) .withNumRetries(10) .build(); diff --git a/metron-platform/metron-parsers/src/test/resources/log4j.properties b/metron-platform/metron-parsers/src/test/resources/log4j.properties new file mode 100644 index 0000000000..d5d33a1dd9 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/log4j.properties @@ -0,0 +1,28 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Root logger option +log4j.rootLogger=WARN, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index dc3069f866..c8728570a4 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -49,11 +49,14 @@ ${global_storm_version} - org.apache.curator + + curator-client + org.apache.curator + junit junit @@ -89,8 +92,32 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 + ${global_kafka_version} + + + log4j + log4j + + + + + org.apache.kafka + kafka-clients + ${global_kafka_version} + + + log4j + log4j + + + + + org.apache.kafka + kafka-clients ${global_kafka_version} + test + test log4j diff --git a/metron-platform/metron-pcap-backend/src/main/flux/pcap/remote.yaml b/metron-platform/metron-pcap-backend/src/main/flux/pcap/remote.yaml index 32f994f67a..272ac1305b 100644 --- a/metron-platform/metron-pcap-backend/src/main/flux/pcap/remote.yaml +++ b/metron-platform/metron-pcap-backend/src/main/flux/pcap/remote.yaml @@ -20,7 +20,7 @@ config: components: - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" - id: "kafkaConfig" diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java index b2f261efa2..d9f6831995 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java @@ -30,9 +30,9 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.log4j.Logger; -import storm.kafka.Callback; -import storm.kafka.EmitContext; -import storm.kafka.PartitionManager; +import org.apache.storm.kafka.Callback; +import org.apache.storm.kafka.EmitContext; +import org.apache.storm.kafka.PartitionManager; import javax.annotation.Nullable; import java.io.Closeable; diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/KafkaToHDFSSpout.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/KafkaToHDFSSpout.java index d87e2c0a51..e4f111323b 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/KafkaToHDFSSpout.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/KafkaToHDFSSpout.java @@ -18,8 +18,8 @@ package org.apache.metron.spout.pcap; -import storm.kafka.Callback; -import storm.kafka.CallbackKafkaSpout; +import org.apache.storm.kafka.Callback; +import org.apache.storm.kafka.CallbackKafkaSpout; public class KafkaToHDFSSpout extends CallbackKafkaSpout { static final long serialVersionUID = 0xDEADBEEFL; diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/SpoutConfig.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/SpoutConfig.java index 99f9229afa..8a76548a0c 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/SpoutConfig.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/SpoutConfig.java @@ -20,7 +20,7 @@ import org.apache.metron.common.utils.timestamp.TimestampConverters; import org.apache.metron.spout.pcap.scheme.TimestampScheme; -import storm.kafka.BrokerHosts; +import org.apache.storm.kafka.BrokerHosts; public class SpoutConfig extends org.apache.metron.common.spout.kafka.SpoutConfig{ diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromKeyScheme.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromKeyScheme.java index 28aae7ad9e..32d2138f96 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromKeyScheme.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromKeyScheme.java @@ -18,10 +18,9 @@ package org.apache.metron.spout.pcap.scheme; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Values; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; import com.google.common.collect.ImmutableList; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.log4j.Logger; @@ -29,8 +28,9 @@ import org.apache.metron.common.utils.timestamp.TimestampConverters; import org.apache.metron.pcap.PcapHelper; import org.apache.metron.spout.pcap.Endianness; -import storm.kafka.KeyValueScheme; +import org.apache.storm.kafka.KeyValueScheme; +import java.nio.ByteBuffer; import java.util.List; public class FromKeyScheme implements KeyValueScheme, KeyConvertible { @@ -39,15 +39,15 @@ public class FromKeyScheme implements KeyValueScheme, KeyConvertible { private TimestampConverter converter = TimestampConverters.MICROSECONDS; private static Endianness endianness = Endianness.getNativeEndianness(); @Override - public List deserializeKeyAndValue(byte[] key, byte[] value) { - Long ts = converter.toNanoseconds(Bytes.toLong(key)); - byte[] packetHeaderized = PcapHelper.addPacketHeader(ts, value, endianness); + public List deserializeKeyAndValue(ByteBuffer key, ByteBuffer value) { + Long ts = converter.toNanoseconds(key.asLongBuffer().get()); + byte[] packetHeaderized = PcapHelper.addPacketHeader(ts, value.array(), endianness); byte[] globalHeaderized= PcapHelper.addGlobalHeader(packetHeaderized, endianness); return new Values(ImmutableList.of(new LongWritable(ts), new BytesWritable(globalHeaderized))); } @Override - public List deserialize(byte[] ser) { + public List deserialize(ByteBuffer ser) { throw new UnsupportedOperationException("Really only interested in deserializing a key and a value"); } diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromPacketScheme.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromPacketScheme.java index 9b94e2b11f..654e4eda12 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromPacketScheme.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromPacketScheme.java @@ -18,9 +18,9 @@ package org.apache.metron.spout.pcap.scheme; -import backtype.storm.spout.MultiScheme; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Values; +import org.apache.storm.spout.MultiScheme; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; import com.google.common.collect.ImmutableList; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; @@ -28,17 +28,18 @@ import org.apache.metron.common.utils.timestamp.TimestampConverter; import org.apache.metron.pcap.PcapHelper; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; public class FromPacketScheme implements MultiScheme,KeyConvertible { private static final Logger LOG = Logger.getLogger(FromPacketScheme.class); @Override - public Iterable> deserialize(byte[] rawValue) { - byte[] value = rawValue; + public Iterable> deserialize(ByteBuffer rawValue) { + byte[] value = rawValue.array(); Long ts = PcapHelper.getTimestamp(value); if(ts != null) { - return ImmutableList.of(new Values(ImmutableList.of(new LongWritable(ts), new BytesWritable(rawValue)))); + return ImmutableList.of(new Values(ImmutableList.of(new LongWritable(ts), new BytesWritable(value)))); } else { return ImmutableList.of(new Values(Collections.EMPTY_LIST)); diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampScheme.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampScheme.java index 3f98c442bf..2be55a9c08 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampScheme.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampScheme.java @@ -18,9 +18,9 @@ package org.apache.metron.spout.pcap.scheme; -import backtype.storm.spout.MultiScheme; +import org.apache.storm.spout.MultiScheme; import org.apache.metron.common.utils.timestamp.TimestampConverter; -import storm.kafka.KeyValueSchemeAsMultiScheme; +import org.apache.storm.kafka.KeyValueSchemeAsMultiScheme; public enum TimestampScheme { FROM_KEY( converter -> new KeyValueSchemeAsMultiScheme(new FromKeyScheme().withTimestampConverter(converter))) diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampSchemeCreator.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampSchemeCreator.java index e88926b877..78aa527096 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampSchemeCreator.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampSchemeCreator.java @@ -18,7 +18,7 @@ package org.apache.metron.spout.pcap.scheme; -import backtype.storm.spout.MultiScheme; +import org.apache.storm.spout.MultiScheme; import org.apache.metron.common.utils.timestamp.TimestampConverter; public interface TimestampSchemeCreator { diff --git a/metron-platform/metron-pcap-backend/src/main/resources/META-INF/LICENSE b/metron-platform/metron-pcap-backend/src/main/resources/META-INF/LICENSE index 0905df5b28..ed6a2522bf 100644 --- a/metron-platform/metron-pcap-backend/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-pcap-backend/src/main/resources/META-INF/LICENSE @@ -219,7 +219,7 @@ This product bundles jaxb-api 2.2.2, which is available under a "Common Developm This product bundles stax-api 1.0-2, which is available under a "Common Development and Distribution License v1.0" license. For details, see https://docs.oracle.com/javase/7/docs/api/javax/xml/stream/package-summary.html This product bundles jopt-simple 3.2, which is available under a "MIT Software License" license. For details, see http://jopt-simple.sourceforge.net This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml index 6b1471249c..4f0ce71e2a 100644 --- a/metron-platform/metron-pcap/pom.xml +++ b/metron-platform/metron-pcap/pom.xml @@ -85,7 +85,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} provided diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapWriter.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapWriter.java index c07a217a78..a67cd1b4be 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapWriter.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.pcap.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.hbase.writer.HBaseWriter; import org.apache.metron.pcap.utils.PcapUtils; import org.json.simple.JSONObject; diff --git a/metron-platform/metron-pcap/src/main/java/storm/kafka/Callback.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/Callback.java similarity index 96% rename from metron-platform/metron-pcap/src/main/java/storm/kafka/Callback.java rename to metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/Callback.java index 4fc1c551a6..8e9622ccda 100644 --- a/metron-platform/metron-pcap/src/main/java/storm/kafka/Callback.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/Callback.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package storm.kafka; +package org.apache.storm.kafka; import java.io.Serializable; import java.util.List; diff --git a/metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackCollector.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackCollector.java similarity index 98% rename from metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackCollector.java rename to metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackCollector.java index 6f2f7fa7a1..30aee30e3d 100644 --- a/metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackCollector.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackCollector.java @@ -15,10 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package storm.kafka; +package org.apache.storm.kafka; -import backtype.storm.spout.ISpoutOutputCollector; -import backtype.storm.spout.SpoutOutputCollector; +import org.apache.storm.spout.SpoutOutputCollector; import java.io.Serializable; import java.util.List; diff --git a/metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackKafkaSpout.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java similarity index 92% rename from metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackKafkaSpout.java rename to metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java index 3828d21b83..661fb0b94d 100644 --- a/metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackKafkaSpout.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java @@ -15,13 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package storm.kafka; +package org.apache.storm.kafka; -import backtype.storm.Config; -import backtype.storm.metric.api.IMetric; -import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.task.TopologyContext; -import storm.kafka.*; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; import java.util.*; @@ -39,10 +36,10 @@ public CallbackKafkaSpout(SpoutConfig spoutConf, Class callb callbackClazz = callback; } - public void initialize() { + public void initialize(TopologyContext context) { _callback = createCallback(callbackClazz); _context = new EmitContext().with(EmitContext.Type.SPOUT_CONFIG, _spoutConfig) - .with(EmitContext.Type.UUID, _uuid) + .with(EmitContext.Type.UUID, context.getStormId()) .with(EmitContext.Type.TOPIC, _spoutConfig.topic); _callback.initialize(_context); } @@ -70,7 +67,7 @@ protected Callback createCallback(Class callbackClass) { @Override public void open(Map conf, final TopologyContext context, final SpoutOutputCollector collector) { if(_callback == null) { - initialize(); + initialize(context); } super.open( conf, context , new CallbackCollector(_callback, collector diff --git a/metron-platform/metron-pcap/src/main/java/storm/kafka/EmitContext.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/EmitContext.java similarity index 98% rename from metron-platform/metron-pcap/src/main/java/storm/kafka/EmitContext.java rename to metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/EmitContext.java index d2d9df76e0..434d884243 100644 --- a/metron-platform/metron-pcap/src/main/java/storm/kafka/EmitContext.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/EmitContext.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package storm.kafka; +package org.apache.storm.kafka; -import backtype.storm.task.TopologyContext; +import org.apache.storm.task.TopologyContext; import java.io.Serializable; import java.util.EnumMap; diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml index 65077e0b5f..d87e860651 100644 --- a/metron-platform/metron-solr/pom.xml +++ b/metron-platform/metron-solr/pom.xml @@ -112,7 +112,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} @@ -121,6 +121,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + org.apache.metron metron-integration-test diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java index 4441b29bdc..50b11ebe25 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java @@ -17,7 +17,9 @@ */ package org.apache.metron.solr.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; +import org.apache.metron.common.configuration.Configurations; +import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.writer.BulkMessageWriter; import org.apache.metron.common.writer.BulkWriterResponse; diff --git a/metron-platform/metron-test-utilities/pom.xml b/metron-platform/metron-test-utilities/pom.xml index 4796667615..f6015c44cd 100644 --- a/metron-platform/metron-test-utilities/pom.xml +++ b/metron-platform/metron-test-utilities/pom.xml @@ -69,7 +69,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/BaseBoltTest.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/BaseBoltTest.java index a655e14f19..3faa1e1332 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/BaseBoltTest.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/BaseBoltTest.java @@ -17,11 +17,11 @@ */ package org.apache.metron.test.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import org.apache.curator.framework.CuratorFramework; diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/PrintingBolt.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/PrintingBolt.java index 83681c75e7..ff9e0cf558 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/PrintingBolt.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/PrintingBolt.java @@ -20,11 +20,11 @@ import java.util.Map; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichBolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichBolt; +import org.apache.storm.tuple.Tuple; @SuppressWarnings("serial") public class PrintingBolt extends BaseRichBolt { diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java index a36d99d256..58b6fefc25 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java @@ -29,13 +29,13 @@ import org.apache.metron.test.converters.IConverter; import org.apache.metron.test.filereaders.FileReader; -import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichSpout; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Values; -import backtype.storm.utils.Utils; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichSpout; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; +import org.apache.storm.utils.Utils; public class GenericInternalTestSpout extends BaseRichSpout { diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml index 1859c7f94d..15b169854c 100644 --- a/metron-platform/metron-writer/pom.xml +++ b/metron-platform/metron-writer/pom.xml @@ -74,7 +74,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} provided diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java index 897ef422ac..738ccacca4 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java @@ -18,8 +18,8 @@ package org.apache.metron.writer; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.Iterables; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java index c6d220e6c3..e323056848 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java index 3b1321b2a8..e3e8150780 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java @@ -18,7 +18,7 @@ package org.apache.metron.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.Iterables; import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java index 161f4bc4ab..a60ddae7e8 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java @@ -17,11 +17,11 @@ */ package org.apache.metron.writer.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt; import org.apache.metron.common.configuration.writer.EnrichmentWriterConfiguration; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java index e4d56d923a..3af0a93569 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java @@ -17,7 +17,8 @@ */ package org.apache.metron.writer.hdfs; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; +import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.writer.BulkMessageWriter; import org.apache.metron.common.writer.BulkWriterResponse; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java index ae0242dd0c..fe9e3e376b 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java @@ -18,7 +18,7 @@ package org.apache.metron.writer.hdfs; -import backtype.storm.task.TopologyContext; +import org.apache.storm.task.TopologyContext; import org.apache.storm.hdfs.bolt.format.FileNameFormat; import java.util.Map; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java index 4a96856510..9cc86803d6 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.writer.kafka; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.base.Joiner; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java index 49bab6fbad..99c825fa4f 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java @@ -18,7 +18,7 @@ package org.apache.metron.writer.message; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; public interface MessageGetter { diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java index f25ea62528..da4a5495e1 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java @@ -19,7 +19,7 @@ package org.apache.metron.writer.message; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; public enum MessageGetters implements MessageGetter{ diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java index 70cfab2681..fdd5fb8081 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java @@ -18,7 +18,7 @@ package org.apache.metron.writer.message; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; public class NamedMessageGetter implements MessageGetter { diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java index 16c01732ac..99a837869b 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java @@ -18,7 +18,7 @@ package org.apache.metron.writer.message; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.utils.JSONUtils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; diff --git a/pom.xml b/pom.xml index 4d6adb38cc..646c05e4de 100644 --- a/pom.xml +++ b/pom.xml @@ -32,10 +32,10 @@ @ApacheMetron 3.7 - 0.10.0 - 0.10.0 + 1.0.1 + 1.0.1 1.7.1 - 0.8.2.2 + 0.10.0.1 2.7.1 1.1.1 1.5.2