From dbf7ff19072cb44df66928920a70e0f1f1f8e4e8 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 11 Oct 2016 09:58:39 -0400 Subject: [PATCH 1/7] Initial move to Storm 1.0.1 --- .../hbase/SaltyRowKeyBuilderTest.java | 2 +- .../profiler/bolt/ProfileBuilderBolt.java | 16 +++++++------- .../profiler/bolt/ProfileHBaseMapper.java | 2 +- .../profiler/bolt/ProfileSplitterBolt.java | 12 +++++------ .../profiler/bolt/ProfileBuilderBoltTest.java | 6 +++--- .../profiler/bolt/ProfileHBaseMapperTest.java | 2 +- .../bolt/ProfileSplitterBoltTest.java | 4 ++-- .../metron/common/bolt/ConfiguredBolt.java | 6 +++--- .../common/spout/kafka/SpoutConfig.java | 4 ++-- .../spout/kafka/SpoutConfigFunction.java | 2 +- .../spout/kafka/SpoutConfigOptions.java | 2 +- .../metron/common/utils/ErrorUtils.java | 4 ++-- .../common/writer/BulkMessageWriter.java | 4 +++- .../metron/common/writer/MessageWriter.java | 2 +- .../bolt/ConfiguredEnrichmentBoltTest.java | 4 ++-- .../common/bolt/ConfiguredParserBoltTest.java | 4 ++-- .../writer/ElasticsearchWriter.java | 2 +- .../enrichment/bolt/EnrichmentJoinBolt.java | 2 +- .../bolt/EnrichmentSplitterBolt.java | 6 +++--- .../bolt/GenericEnrichmentBolt.java | 12 +++++------ .../metron/enrichment/bolt/JoinBolt.java | 12 +++++------ .../metron/enrichment/bolt/SplitBolt.java | 12 +++++------ .../enrichment/bolt/ThreatIntelJoinBolt.java | 2 +- .../writer/SimpleHbaseEnrichmentWriter.java | 2 +- .../bolt/BulkMessageWriterBoltTest.java | 4 ++-- .../bolt/GenericEnrichmentBoltTest.java | 2 +- .../metron/enrichment/bolt/JoinBoltTest.java | 4 ++-- .../metron/enrichment/bolt/SplitBoltTest.java | 8 +++---- .../apache/metron/hbase/HTableConnector.java | 2 +- .../apache/metron/hbase/TupleTableConfig.java | 2 +- .../apache/metron/hbase/bolt/BatchHelper.java | 6 +++--- .../apache/metron/hbase/bolt/HBaseBolt.java | 12 +++++------ .../metron/hbase/bolt/mapper/HBaseMapper.java | 2 +- .../metron/hbase/writer/HBaseWriter.java | 2 +- .../org/apache/metron/hbase/WidgetMapper.java | 2 +- .../metron/hbase/bolt/HBaseBoltTest.java | 4 ++-- .../metron/hbase/client/HBaseClientTest.java | 2 +- .../components/FluxTopologyComponent.java | 14 ++++++------- .../metron/parsers/bolt/ParserBolt.java | 12 +++++------ .../metron/parsers/bolt/WriterBolt.java | 10 ++++----- .../metron/parsers/bolt/WriterHandler.java | 4 ++-- .../topology/ParserTopologyBuilder.java | 6 +++--- .../parsers/topology/ParserTopologyCLI.java | 11 +++++----- .../metron/parsers/topology/config/Arg.java | 2 +- .../topology/config/ConfigHandlers.java | 2 +- .../metron/parsers/bolt/ParserBoltTest.java | 4 ++-- .../metron/parsers/bolt/WriterBoltTest.java | 8 ++++--- .../integration/ParserIntegrationTest.java | 2 +- .../components/ParserTopologyComponent.java | 6 +++--- .../topology/ParserTopologyCLITest.java | 2 +- ...eHbaseEnrichmentWriterIntegrationTest.java | 2 +- .../WriterBoltIntegrationTest.java | 2 +- .../metron/spout/pcap/HDFSWriterCallback.java | 6 +++--- .../metron/spout/pcap/KafkaToHDFSSpout.java | 4 ++-- .../apache/metron/spout/pcap/SpoutConfig.java | 2 +- .../spout/pcap/scheme/FromKeyScheme.java | 16 +++++++------- .../spout/pcap/scheme/FromPacketScheme.java | 13 ++++++------ .../spout/pcap/scheme/TimestampScheme.java | 4 ++-- .../pcap/scheme/TimestampSchemeCreator.java | 2 +- .../PcapTopologyIntegrationTest.java | 2 +- .../apache/metron/pcap/writer/PcapWriter.java | 2 +- .../apache}/storm/kafka/Callback.java | 2 +- .../storm/kafka/CallbackCollector.java | 5 ++--- .../storm/kafka/CallbackKafkaSpout.java | 21 +++++++++---------- .../apache}/storm/kafka/EmitContext.java | 4 ++-- .../apache/metron/solr/writer/SolrWriter.java | 4 +++- .../apache/metron/test/bolt/BaseBoltTest.java | 10 ++++----- .../apache/metron/test/bolt/PrintingBolt.java | 10 ++++----- .../test/spouts/GenericInternalTestSpout.java | 14 ++++++------- .../metron/writer/BulkWriterComponent.java | 4 ++-- .../org/apache/metron/writer/NoopWriter.java | 2 +- .../metron/writer/WriterToBulkWriter.java | 2 +- .../writer/bolt/BulkMessageWriterBolt.java | 10 ++++----- .../apache/metron/writer/hdfs/HdfsWriter.java | 3 ++- .../writer/hdfs/SourceFileNameFormat.java | 2 +- .../metron/writer/kafka/KafkaWriter.java | 2 +- .../metron/writer/message/MessageGetter.java | 2 +- .../metron/writer/message/MessageGetters.java | 2 +- .../writer/message/NamedMessageGetter.java | 2 +- .../writer/message/RawMessageGetter.java | 2 +- pom.xml | 4 ++-- 81 files changed, 214 insertions(+), 209 deletions(-) rename metron-platform/metron-pcap/src/main/java/{ => org/apache}/storm/kafka/Callback.java (96%) rename metron-platform/metron-pcap/src/main/java/{ => org/apache}/storm/kafka/CallbackCollector.java (98%) rename metron-platform/metron-pcap/src/main/java/{ => org/apache}/storm/kafka/CallbackKafkaSpout.java (82%) rename metron-platform/metron-pcap/src/main/java/{ => org/apache}/storm/kafka/EmitContext.java (98%) diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java index d59f38d96a..ccbb09cea5 100644 --- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java +++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java @@ -20,7 +20,7 @@ package org.apache.metron.profiler.hbase; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.ProfilePeriod; import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java index 15c803ca6e..fa732e2b5f 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java @@ -20,14 +20,14 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.Config; -import backtype.storm.Constants; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.Config; +import org.apache.storm.Constants; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.metron.common.bolt.ConfiguredProfilerBolt; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.dsl.Context; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java index d6377e4f39..28d0530bc0 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java @@ -20,7 +20,7 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.commons.beanutils.BeanMap; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.dsl.ParseException; diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java index 5d97e4d8bf..c784c8e27d 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java @@ -20,12 +20,12 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.commons.lang.StringUtils; import org.apache.metron.common.bolt.ConfiguredProfilerBolt; import org.apache.metron.common.configuration.profiler.ProfileConfig; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java index caeddee7e7..48ff663372 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java @@ -20,9 +20,9 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.Constants; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.Constants; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.utils.JSONUtils; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java index 97caff95e3..f2fa2f63a4 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java @@ -20,7 +20,7 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.hbase.RowKeyBuilder; diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java index 31ff279837..0ac1c332ec 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java @@ -20,8 +20,8 @@ package org.apache.metron.profiler.bolt; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.profiler.stellar.DefaultStellarExecutor; import org.apache.metron.test.bolt.BaseBoltTest; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java index 9318bf0b86..cef4affa4a 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java @@ -17,9 +17,9 @@ */ package org.apache.metron.common.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.base.BaseRichBolt; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.base.BaseRichBolt; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java index 8f0a6afbfe..c26d1f5632 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java @@ -18,9 +18,9 @@ package org.apache.metron.common.spout.kafka; -import storm.kafka.BrokerHosts; +import org.apache.storm.kafka.BrokerHosts; -public class SpoutConfig extends storm.kafka.SpoutConfig { +public class SpoutConfig extends org.apache.storm.kafka.SpoutConfig { public static enum Offset { BEGINNING, END, WHERE_I_LEFT_OFF; } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java index e8abd60756..5aec998a62 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java @@ -19,6 +19,6 @@ package org.apache.metron.common.spout.kafka; public interface SpoutConfigFunction { - void configure(storm.kafka.SpoutConfig config, Object val); + void configure(org.apache.storm.kafka.SpoutConfig config, Object val); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java index 807694e15b..7d1c7c331f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java @@ -20,7 +20,7 @@ import com.google.common.base.Joiner; import org.apache.metron.common.utils.ConversionUtils; -import storm.kafka.SpoutConfig; +import org.apache.storm.kafka.SpoutConfig; import java.util.EnumMap; import java.util.Map; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java index 92e0ab91cc..3b3f42683f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java @@ -24,8 +24,8 @@ import java.util.List; import java.util.Optional; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Values; import org.apache.commons.beanutils.Converter; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.ArrayUtils; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java index 0e1aa90525..64e16b1b3e 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java @@ -17,7 +17,9 @@ */ package org.apache.metron.common.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; +import org.apache.metron.common.configuration.Configurations; +import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.writer.WriterConfiguration; import java.io.Serializable; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java index 74339e95da..1ae54b15be 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.common.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.Configurations; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java index 64fdace475..fcd00fe65b 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.common.bolt; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Tuple; import org.apache.curator.test.TestingServer; import org.apache.metron.common.Constants; import org.apache.metron.TestConstants; diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java index a1bbc138af..ea864b9359 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.common.bolt; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Tuple; import org.apache.curator.test.TestingServer; import org.apache.metron.TestConstants; import org.apache.metron.common.Constants; diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java index d8f34a9370..7360649b1a 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.elasticsearch.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java index de6967bcfd..2adf4301a9 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java @@ -17,7 +17,7 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; +import org.apache.storm.task.TopologyContext; import com.google.common.base.Joiner; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java index af59dcd50c..1ec4252b8a 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java @@ -17,9 +17,9 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java index 059817a623..51d98b9593 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java @@ -18,12 +18,12 @@ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java index e0c96a2976..b31418c334 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java @@ -17,12 +17,12 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import com.google.common.base.Joiner; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java index e4b34a0c08..953e6a6dff 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java @@ -17,12 +17,12 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.metron.common.bolt.ConfiguredBolt; import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java index 01e5a9fbaf..fc92bff447 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java @@ -17,7 +17,7 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; +import org.apache.storm.task.TopologyContext; import com.google.common.base.Joiner; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java index 0314354167..13d88d5b6b 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java @@ -18,7 +18,7 @@ package org.apache.metron.enrichment.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import org.apache.hadoop.conf.Configuration; diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java index a692db4ed3..c53e053e0e 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java index 3b4fed0562..8fd3e73ea4 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java @@ -17,7 +17,7 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.tuple.Values; +import org.apache.storm.tuple.Values; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.TestConstants; diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java index 931297d808..5b06d336fa 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; -import backtype.storm.tuple.Values; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.test.bolt.BaseEnrichmentBoltTest; import org.json.simple.JSONObject; diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java index 6653b6d136..d0ca475156 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java @@ -17,10 +17,10 @@ */ package org.apache.metron.enrichment.bolt; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import com.google.common.collect.ImmutableList; import junit.framework.Assert; import org.apache.metron.common.configuration.ConfigurationType; diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableConnector.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableConnector.java index ace4d8095e..504826e0dc 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableConnector.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableConnector.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.log4j.Logger; -import backtype.storm.generated.Bolt; +import org.apache.storm.generated.Bolt; /** * HTable connector for Storm {@link Bolt} diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TupleTableConfig.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TupleTableConfig.java index 8257d8aca1..e491184551 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TupleTableConfig.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TupleTableConfig.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.log4j.Logger; /** diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/BatchHelper.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/BatchHelper.java index e7cf2cfd50..9e3bbe0363 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/BatchHelper.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/BatchHelper.java @@ -20,9 +20,9 @@ package org.apache.metron.hbase.bolt; -import backtype.storm.Constants; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Tuple; +import org.apache.storm.Constants; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/HBaseBolt.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/HBaseBolt.java index ede1d8c9a5..6fc6b0af57 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/HBaseBolt.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/HBaseBolt.java @@ -20,12 +20,12 @@ package org.apache.metron.hbase.bolt; -import backtype.storm.Config; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichBolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.Config; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichBolt; +import org.apache.storm.tuple.Tuple; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Durability; diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/mapper/HBaseMapper.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/mapper/HBaseMapper.java index 6cae1b4fbb..6c7e5fbf3b 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/mapper/HBaseMapper.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/bolt/mapper/HBaseMapper.java @@ -20,7 +20,7 @@ package org.apache.metron.hbase.bolt.mapper; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import java.io.Serializable; import java.util.Optional; diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java index e53d38966c..f46cfde9c0 100644 --- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java +++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.hbase.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTableInterface; diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/WidgetMapper.java b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/WidgetMapper.java index 4f97a5b599..6d56a4b405 100644 --- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/WidgetMapper.java +++ b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/WidgetMapper.java @@ -20,7 +20,7 @@ package org.apache.metron.hbase; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.hbase.bolt.mapper.ColumnList; import org.apache.metron.hbase.bolt.mapper.HBaseMapper; diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java index 621720e32e..f445b1ba44 100644 --- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java +++ b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java @@ -20,8 +20,8 @@ package org.apache.metron.hbase.bolt; -import backtype.storm.Constants; -import backtype.storm.tuple.Tuple; +import org.apache.storm.Constants; +import org.apache.storm.tuple.Tuple; import org.apache.metron.hbase.Widget; import org.apache.metron.hbase.WidgetMapper; import org.apache.metron.hbase.client.HBaseClient; diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java index 00a11266e3..27544c009d 100644 --- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java +++ b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java @@ -20,7 +20,7 @@ package org.apache.metron.hbase.client; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java index 75be2d7d10..d48a981118 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java @@ -17,17 +17,17 @@ */ package org.apache.metron.integration.components; -import backtype.storm.Config; -import backtype.storm.LocalCluster; -import backtype.storm.generated.StormTopology; -import backtype.storm.generated.TopologyInfo; +import org.apache.storm.Config; +import org.apache.storm.LocalCluster; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.generated.TopologyInfo; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; import org.apache.storm.flux.FluxBuilder; import org.apache.storm.flux.model.ExecutionContext; import org.apache.storm.flux.model.TopologyDef; import org.apache.storm.flux.parser.FluxParser; -import org.apache.thrift7.TException; +import org.apache.storm.thrift.TException; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,11 +115,11 @@ public void stop() { } } - public void submitTopology() throws NoSuchMethodException, IOException, InstantiationException, TException, IllegalAccessException, InvocationTargetException, ClassNotFoundException { + public void submitTopology() throws NoSuchMethodException, IOException, InstantiationException, TException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, NoSuchFieldException { startTopology(getTopologyName(), getTopologyLocation(), getTopologyProperties()); } - private void startTopology(String topologyName, File topologyLoc, Properties properties) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, TException { + private void startTopology(String topologyName, File topologyLoc, Properties properties) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, TException, NoSuchFieldException{ TopologyDef topologyDef = loadYaml(topologyName, topologyLoc, properties); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 325209f880..476df5bc47 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -17,12 +17,12 @@ */ package org.apache.metron.parsers.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredParserBolt; import org.apache.metron.common.configuration.FieldValidator; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java index 380f760b7f..8eb065614d 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java @@ -18,11 +18,11 @@ package org.apache.metron.parsers.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichBolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichBolt; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.utils.ErrorUtils; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java index 24d803f94e..38425b5891 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java @@ -18,8 +18,8 @@ package org.apache.metron.parsers.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java index d5ec0558f7..5f1927e9bc 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java @@ -17,7 +17,7 @@ */ package org.apache.metron.parsers.topology; -import backtype.storm.topology.TopologyBuilder; +import org.apache.storm.topology.TopologyBuilder; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.ConfigurationsUtils; @@ -36,8 +36,8 @@ import org.apache.metron.writer.AbstractWriter; import org.apache.metron.writer.kafka.KafkaWriter; import org.json.simple.JSONObject; -import storm.kafka.KafkaSpout; -import storm.kafka.ZkHosts; +import org.apache.storm.kafka.KafkaSpout; +import org.apache.storm.kafka.ZkHosts; import java.util.EnumMap; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java index 0ebec6c1c7..5ea561c709 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java @@ -17,12 +17,11 @@ */ package org.apache.metron.parsers.topology; -import backtype.storm.Config; -import backtype.storm.ConfigValidation; -import backtype.storm.LocalCluster; -import backtype.storm.StormSubmitter; -import backtype.storm.topology.TopologyBuilder; -import backtype.storm.utils.Utils; +import org.apache.storm.Config; +import org.apache.storm.LocalCluster; +import org.apache.storm.StormSubmitter; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.utils.Utils; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Joiner; import org.apache.commons.cli.*; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java index add11b7d65..c746a56f93 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java @@ -18,7 +18,7 @@ package org.apache.metron.parsers.topology.config; -import backtype.storm.Config; +import org.apache.storm.Config; public class Arg { private Config config; diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java index f982e29489..f6c04a6c5d 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java @@ -18,7 +18,7 @@ package org.apache.metron.parsers.topology.config; -import backtype.storm.Config; +import org.apache.storm.Config; import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.io.FileUtils; import org.apache.metron.common.utils.JSONUtils; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index 5d938384fc..cdebbc3fd2 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -19,8 +19,8 @@ import org.apache.metron.common.configuration.SensorParserConfig; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.ImmutableList; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java index 083a4e3c6c..e8e2653d77 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java @@ -17,9 +17,11 @@ */ package org.apache.metron.parsers.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.tuple.Tuple; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java index b3d09d07d5..14fe748690 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java @@ -62,7 +62,7 @@ public void test() throws Exception { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("kafka", kafkaComponent) .withComponent("config", configUploadComponent) - .withComponent("storm", parserTopologyComponent) + .withComponent("org/apache/storm", parserTopologyComponent) .withMillisecondsBetweenAttempts(5000) .withNumRetries(10) .build(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java index f639e0269e..80b6ebdb87 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java @@ -17,9 +17,9 @@ */ package org.apache.metron.parsers.integration.components; -import backtype.storm.Config; -import backtype.storm.LocalCluster; -import backtype.storm.topology.TopologyBuilder; +import org.apache.storm.Config; +import org.apache.storm.LocalCluster; +import org.apache.storm.topology.TopologyBuilder; import org.apache.metron.common.spout.kafka.SpoutConfig; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java index 2940bfe416..83a0b6acca 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java @@ -18,7 +18,7 @@ package org.apache.metron.parsers.topology; -import backtype.storm.Config; +import org.apache.storm.Config; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.cli.CommandLine; diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java index 2f359ef521..6dab0c5c46 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java @@ -96,7 +96,7 @@ public void test() throws UnableToStartException, IOException { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("kafka", kafkaComponent) .withComponent("config", configUploadComponent) - .withComponent("storm", parserTopologyComponent) + .withComponent("org/apache/storm", parserTopologyComponent) .withMillisecondsBetweenAttempts(5000) .withNumRetries(10) .build(); diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java index b9377f3ff2..f97619fcef 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java @@ -112,7 +112,7 @@ public void test() throws UnableToStartException, IOException { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("kafka", kafkaComponent) .withComponent("config", configUploadComponent) - .withComponent("storm", parserTopologyComponent) + .withComponent("org/apache/storm", parserTopologyComponent) .withMillisecondsBetweenAttempts(5000) .withNumRetries(10) .build(); diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java index b2f261efa2..d9f6831995 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java @@ -30,9 +30,9 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.log4j.Logger; -import storm.kafka.Callback; -import storm.kafka.EmitContext; -import storm.kafka.PartitionManager; +import org.apache.storm.kafka.Callback; +import org.apache.storm.kafka.EmitContext; +import org.apache.storm.kafka.PartitionManager; import javax.annotation.Nullable; import java.io.Closeable; diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/KafkaToHDFSSpout.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/KafkaToHDFSSpout.java index d87e2c0a51..e4f111323b 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/KafkaToHDFSSpout.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/KafkaToHDFSSpout.java @@ -18,8 +18,8 @@ package org.apache.metron.spout.pcap; -import storm.kafka.Callback; -import storm.kafka.CallbackKafkaSpout; +import org.apache.storm.kafka.Callback; +import org.apache.storm.kafka.CallbackKafkaSpout; public class KafkaToHDFSSpout extends CallbackKafkaSpout { static final long serialVersionUID = 0xDEADBEEFL; diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/SpoutConfig.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/SpoutConfig.java index 99f9229afa..8a76548a0c 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/SpoutConfig.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/SpoutConfig.java @@ -20,7 +20,7 @@ import org.apache.metron.common.utils.timestamp.TimestampConverters; import org.apache.metron.spout.pcap.scheme.TimestampScheme; -import storm.kafka.BrokerHosts; +import org.apache.storm.kafka.BrokerHosts; public class SpoutConfig extends org.apache.metron.common.spout.kafka.SpoutConfig{ diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromKeyScheme.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromKeyScheme.java index 28aae7ad9e..32d2138f96 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromKeyScheme.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromKeyScheme.java @@ -18,10 +18,9 @@ package org.apache.metron.spout.pcap.scheme; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Values; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; import com.google.common.collect.ImmutableList; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.log4j.Logger; @@ -29,8 +28,9 @@ import org.apache.metron.common.utils.timestamp.TimestampConverters; import org.apache.metron.pcap.PcapHelper; import org.apache.metron.spout.pcap.Endianness; -import storm.kafka.KeyValueScheme; +import org.apache.storm.kafka.KeyValueScheme; +import java.nio.ByteBuffer; import java.util.List; public class FromKeyScheme implements KeyValueScheme, KeyConvertible { @@ -39,15 +39,15 @@ public class FromKeyScheme implements KeyValueScheme, KeyConvertible { private TimestampConverter converter = TimestampConverters.MICROSECONDS; private static Endianness endianness = Endianness.getNativeEndianness(); @Override - public List deserializeKeyAndValue(byte[] key, byte[] value) { - Long ts = converter.toNanoseconds(Bytes.toLong(key)); - byte[] packetHeaderized = PcapHelper.addPacketHeader(ts, value, endianness); + public List deserializeKeyAndValue(ByteBuffer key, ByteBuffer value) { + Long ts = converter.toNanoseconds(key.asLongBuffer().get()); + byte[] packetHeaderized = PcapHelper.addPacketHeader(ts, value.array(), endianness); byte[] globalHeaderized= PcapHelper.addGlobalHeader(packetHeaderized, endianness); return new Values(ImmutableList.of(new LongWritable(ts), new BytesWritable(globalHeaderized))); } @Override - public List deserialize(byte[] ser) { + public List deserialize(ByteBuffer ser) { throw new UnsupportedOperationException("Really only interested in deserializing a key and a value"); } diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromPacketScheme.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromPacketScheme.java index 9b94e2b11f..654e4eda12 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromPacketScheme.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/FromPacketScheme.java @@ -18,9 +18,9 @@ package org.apache.metron.spout.pcap.scheme; -import backtype.storm.spout.MultiScheme; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Values; +import org.apache.storm.spout.MultiScheme; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; import com.google.common.collect.ImmutableList; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; @@ -28,17 +28,18 @@ import org.apache.metron.common.utils.timestamp.TimestampConverter; import org.apache.metron.pcap.PcapHelper; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; public class FromPacketScheme implements MultiScheme,KeyConvertible { private static final Logger LOG = Logger.getLogger(FromPacketScheme.class); @Override - public Iterable> deserialize(byte[] rawValue) { - byte[] value = rawValue; + public Iterable> deserialize(ByteBuffer rawValue) { + byte[] value = rawValue.array(); Long ts = PcapHelper.getTimestamp(value); if(ts != null) { - return ImmutableList.of(new Values(ImmutableList.of(new LongWritable(ts), new BytesWritable(rawValue)))); + return ImmutableList.of(new Values(ImmutableList.of(new LongWritable(ts), new BytesWritable(value)))); } else { return ImmutableList.of(new Values(Collections.EMPTY_LIST)); diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampScheme.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampScheme.java index 3f98c442bf..2be55a9c08 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampScheme.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampScheme.java @@ -18,9 +18,9 @@ package org.apache.metron.spout.pcap.scheme; -import backtype.storm.spout.MultiScheme; +import org.apache.storm.spout.MultiScheme; import org.apache.metron.common.utils.timestamp.TimestampConverter; -import storm.kafka.KeyValueSchemeAsMultiScheme; +import org.apache.storm.kafka.KeyValueSchemeAsMultiScheme; public enum TimestampScheme { FROM_KEY( converter -> new KeyValueSchemeAsMultiScheme(new FromKeyScheme().withTimestampConverter(converter))) diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampSchemeCreator.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampSchemeCreator.java index e88926b877..78aa527096 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampSchemeCreator.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/scheme/TimestampSchemeCreator.java @@ -18,7 +18,7 @@ package org.apache.metron.spout.pcap.scheme; -import backtype.storm.spout.MultiScheme; +import org.apache.storm.spout.MultiScheme; import org.apache.metron.common.utils.timestamp.TimestampConverter; public interface TimestampSchemeCreator { diff --git a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java index fcdb108d79..77b5f9db69 100644 --- a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java +++ b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java @@ -240,7 +240,7 @@ public Void apply(@Nullable KafkaWithZKComponent kafkaWithZKComponent) { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("mr", mr) .withComponent("kafka", kafkaComponent) - .withComponent("storm", fluxComponent) + .withComponent("org/apache/storm", fluxComponent) .withMaxTimeMS(-1) .withMillisecondsBetweenAttempts(2000) .withNumRetries(10) diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapWriter.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapWriter.java index c07a217a78..a67cd1b4be 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapWriter.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.pcap.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.hbase.writer.HBaseWriter; import org.apache.metron.pcap.utils.PcapUtils; import org.json.simple.JSONObject; diff --git a/metron-platform/metron-pcap/src/main/java/storm/kafka/Callback.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/Callback.java similarity index 96% rename from metron-platform/metron-pcap/src/main/java/storm/kafka/Callback.java rename to metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/Callback.java index 4fc1c551a6..8e9622ccda 100644 --- a/metron-platform/metron-pcap/src/main/java/storm/kafka/Callback.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/Callback.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package storm.kafka; +package org.apache.storm.kafka; import java.io.Serializable; import java.util.List; diff --git a/metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackCollector.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackCollector.java similarity index 98% rename from metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackCollector.java rename to metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackCollector.java index 6f2f7fa7a1..30aee30e3d 100644 --- a/metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackCollector.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackCollector.java @@ -15,10 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package storm.kafka; +package org.apache.storm.kafka; -import backtype.storm.spout.ISpoutOutputCollector; -import backtype.storm.spout.SpoutOutputCollector; +import org.apache.storm.spout.SpoutOutputCollector; import java.io.Serializable; import java.util.List; diff --git a/metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackKafkaSpout.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java similarity index 82% rename from metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackKafkaSpout.java rename to metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java index 3828d21b83..b7bcb58d5a 100644 --- a/metron-platform/metron-pcap/src/main/java/storm/kafka/CallbackKafkaSpout.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java @@ -15,13 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package storm.kafka; +package org.apache.storm.kafka; -import backtype.storm.Config; -import backtype.storm.metric.api.IMetric; -import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.task.TopologyContext; -import storm.kafka.*; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; import java.util.*; @@ -39,11 +36,13 @@ public CallbackKafkaSpout(SpoutConfig spoutConf, Class callb callbackClazz = callback; } - public void initialize() { + public void initialize(TopologyContext context) { _callback = createCallback(callbackClazz); - _context = new EmitContext().with(EmitContext.Type.SPOUT_CONFIG, _spoutConfig) - .with(EmitContext.Type.UUID, _uuid) - .with(EmitContext.Type.TOPIC, _spoutConfig.topic); + //TODO figure out if there's a better way to handle this now +// _context = new EmitContext().with(EmitContext.Type.SPOUT_CONFIG, _spoutConfig) +// .with(EmitContext.Type.UUID, context.getStormId()) +// .with(EmitContext.Type.TOPIC, _spoutConfig.topic); + _context = new EmitContext().with(EmitContext.Type.TOPOLOGY_CONTEXT, context); _callback.initialize(_context); } @@ -70,7 +69,7 @@ protected Callback createCallback(Class callbackClass) { @Override public void open(Map conf, final TopologyContext context, final SpoutOutputCollector collector) { if(_callback == null) { - initialize(); + initialize(context); } super.open( conf, context , new CallbackCollector(_callback, collector diff --git a/metron-platform/metron-pcap/src/main/java/storm/kafka/EmitContext.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/EmitContext.java similarity index 98% rename from metron-platform/metron-pcap/src/main/java/storm/kafka/EmitContext.java rename to metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/EmitContext.java index d2d9df76e0..434d884243 100644 --- a/metron-platform/metron-pcap/src/main/java/storm/kafka/EmitContext.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/EmitContext.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package storm.kafka; +package org.apache.storm.kafka; -import backtype.storm.task.TopologyContext; +import org.apache.storm.task.TopologyContext; import java.io.Serializable; import java.util.EnumMap; diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java index 4441b29bdc..50b11ebe25 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java @@ -17,7 +17,9 @@ */ package org.apache.metron.solr.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; +import org.apache.metron.common.configuration.Configurations; +import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.writer.BulkMessageWriter; import org.apache.metron.common.writer.BulkWriterResponse; diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/BaseBoltTest.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/BaseBoltTest.java index a655e14f19..3faa1e1332 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/BaseBoltTest.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/BaseBoltTest.java @@ -17,11 +17,11 @@ */ package org.apache.metron.test.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import org.apache.curator.framework.CuratorFramework; diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/PrintingBolt.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/PrintingBolt.java index 83681c75e7..ff9e0cf558 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/PrintingBolt.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/bolt/PrintingBolt.java @@ -20,11 +20,11 @@ import java.util.Map; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichBolt; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichBolt; +import org.apache.storm.tuple.Tuple; @SuppressWarnings("serial") public class PrintingBolt extends BaseRichBolt { diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java index a36d99d256..58b6fefc25 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java @@ -29,13 +29,13 @@ import org.apache.metron.test.converters.IConverter; import org.apache.metron.test.filereaders.FileReader; -import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichSpout; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Values; -import backtype.storm.utils.Utils; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichSpout; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; +import org.apache.storm.utils.Utils; public class GenericInternalTestSpout extends BaseRichSpout { diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java index 897ef422ac..738ccacca4 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java @@ -18,8 +18,8 @@ package org.apache.metron.writer; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.Iterables; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java index c6d220e6c3..e323056848 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java index 3b1321b2a8..e3e8150780 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java @@ -18,7 +18,7 @@ package org.apache.metron.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.Iterables; import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade; import org.apache.metron.common.configuration.writer.WriterConfiguration; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java index 161f4bc4ab..a60ddae7e8 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java @@ -17,11 +17,11 @@ */ package org.apache.metron.writer.bolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt; import org.apache.metron.common.configuration.writer.EnrichmentWriterConfiguration; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java index e4d56d923a..3af0a93569 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java @@ -17,7 +17,8 @@ */ package org.apache.metron.writer.hdfs; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; +import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.writer.BulkMessageWriter; import org.apache.metron.common.writer.BulkWriterResponse; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java index ae0242dd0c..fe9e3e376b 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java @@ -18,7 +18,7 @@ package org.apache.metron.writer.hdfs; -import backtype.storm.task.TopologyContext; +import org.apache.storm.task.TopologyContext; import org.apache.storm.hdfs.bolt.format.FileNameFormat; import java.util.Map; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java index 4a96856510..9cc86803d6 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java @@ -17,7 +17,7 @@ */ package org.apache.metron.writer.kafka; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.base.Joiner; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java index 49bab6fbad..99c825fa4f 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java @@ -18,7 +18,7 @@ package org.apache.metron.writer.message; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; public interface MessageGetter { diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java index f25ea62528..da4a5495e1 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java @@ -19,7 +19,7 @@ package org.apache.metron.writer.message; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; public enum MessageGetters implements MessageGetter{ diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java index 70cfab2681..fdd5fb8081 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java @@ -18,7 +18,7 @@ package org.apache.metron.writer.message; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; public class NamedMessageGetter implements MessageGetter { diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java index 16c01732ac..99a837869b 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java @@ -18,7 +18,7 @@ package org.apache.metron.writer.message; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import org.apache.metron.common.utils.JSONUtils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; diff --git a/pom.xml b/pom.xml index 4d6adb38cc..29c103e3de 100644 --- a/pom.xml +++ b/pom.xml @@ -32,8 +32,8 @@ @ApacheMetron 3.7 - 0.10.0 - 0.10.0 + 1.0.1 + 1.0.1 1.7.1 0.8.2.2 2.7.1 From 9796bf36fc4a19fbed5276166c305cb8fb815e00 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Wed, 12 Oct 2016 11:13:45 -0400 Subject: [PATCH 2/7] fixing more imports --- .../org/apache/metron/common/writer/BulkWriterResponse.java | 2 +- .../metron/elasticsearch/writer/ElasticsearchWriterTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java index 2992358578..c4fa881d3d 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java @@ -18,7 +18,7 @@ package org.apache.metron.common.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java index 51e7e0a2df..9aff560064 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java @@ -18,7 +18,7 @@ package org.apache.metron.elasticsearch.writer; -import backtype.storm.tuple.Tuple; +import org.apache.storm.tuple.Tuple; import com.google.common.collect.ImmutableList; import org.apache.metron.common.writer.BulkWriterResponse; import org.elasticsearch.action.bulk.BulkItemResponse; From 044b9d7113f3b7d2d367f2f58e40872ea372d75a Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 13 Oct 2016 15:56:27 -0400 Subject: [PATCH 3/7] changes to kakfa. WIP --- metron-platform/metron-api/pom.xml | 2 +- metron-platform/metron-common/pom.xml | 4 +- .../common/spout/kafka/SpoutConfig.java | 118 +++++++++--------- .../spout/kafka/SpoutConfigFunction.java | 2 +- .../spout/kafka/SpoutConfigOptions.java | 5 +- metron-platform/metron-elasticsearch/pom.xml | 2 +- metron-platform/metron-enrichment/pom.xml | 2 +- .../metron-integration-test/pom.xml | 20 ++- .../components/KafkaWithZKComponent.java | 69 ++++++++-- .../wrapper/AdminUtilsWrapper.scala | 33 +++++ .../wrapper/TestUtilsWrapper.scala | 29 +++++ metron-platform/metron-parsers/pom.xml | 2 +- .../src/test/resources/log4j.properties | 28 +++++ metron-platform/metron-pcap-backend/pom.xml | 4 +- metron-platform/metron-pcap/pom.xml | 4 +- metron-platform/metron-solr/pom.xml | 2 +- metron-platform/metron-test-utilities/pom.xml | 2 +- metron-platform/metron-writer/pom.xml | 2 +- 18 files changed, 235 insertions(+), 95 deletions(-) create mode 100644 metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/AdminUtilsWrapper.scala create mode 100644 metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/TestUtilsWrapper.scala create mode 100644 metron-platform/metron-parsers/src/test/resources/log4j.properties diff --git a/metron-platform/metron-api/pom.xml b/metron-platform/metron-api/pom.xml index d600bfddc0..3c9c9e5a64 100644 --- a/metron-platform/metron-api/pom.xml +++ b/metron-platform/metron-api/pom.xml @@ -29,7 +29,7 @@ 3.4.5.2.0.6.0-76 1.2.15 - 0.9.2-incubating + 1.0.1 0.0.7-SNAPSHOT 0.0.5-SNAPSHOT diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index c19a507cd3..581766722d 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -86,7 +86,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} provided @@ -193,7 +193,7 @@ org.apache.storm - storm-kafka + storm-kafka-client ${global_storm_version} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java index c26d1f5632..ad18813a4c 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java @@ -1,59 +1,59 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.spout.kafka; - -import org.apache.storm.kafka.BrokerHosts; - -public class SpoutConfig extends org.apache.storm.kafka.SpoutConfig { - public static enum Offset { - BEGINNING, END, WHERE_I_LEFT_OFF; - } - public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) { - super(hosts, topic, zkRoot, id); - } - - public SpoutConfig from(String offset) { - try { - Offset o = Offset.valueOf(offset); - from(o); - } - catch(IllegalArgumentException iae) { - from(Offset.WHERE_I_LEFT_OFF); - } - ignoreZkOffsets = true; - startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); - return this; - } - - public SpoutConfig from(Offset offset) { - if(offset == Offset.BEGINNING) { - ignoreZkOffsets = true; - startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); - } - else if(offset == Offset.END) { - ignoreZkOffsets = true; - startOffsetTime = kafka.api.OffsetRequest.LatestTime(); - } - else if(offset == Offset.WHERE_I_LEFT_OFF) { - ignoreZkOffsets = false; - startOffsetTime = kafka.api.OffsetRequest.LatestTime(); - } - return this; - } -} +///** +// * Licensed to the Apache Software Foundation (ASF) under one +// * or more contributor license agreements. See the NOTICE file +// * distributed with this work for additional information +// * regarding copyright ownership. The ASF licenses this file +// * to you under the Apache License, Version 2.0 (the +// * "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package org.apache.metron.common.spout.kafka; +// +//import org.apache.storm.kafka.BrokerHosts; +// +//public class SpoutConfig extends org.apache.storm.kafka.spout.KafkaSpoutConfig { +// public static enum Offset { +// BEGINNING, END, WHERE_I_LEFT_OFF; +// } +// public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) { +// super(hosts, topic, zkRoot, id); +// } +// +// public SpoutConfig from(String offset) { +// try { +// Offset o = Offset.valueOf(offset); +// from(o); +// } +// catch(IllegalArgumentException iae) { +// from(Offset.WHERE_I_LEFT_OFF); +// } +// ignoreZkOffsets = true; +// startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); +// return this; +// } +// +// public SpoutConfig from(Offset offset) { +// if(offset == Offset.BEGINNING) { +// ignoreZkOffsets = true; +// startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); +// } +// else if(offset == Offset.END) { +// ignoreZkOffsets = true; +// startOffsetTime = kafka.api.OffsetRequest.LatestTime(); +// } +// else if(offset == Offset.WHERE_I_LEFT_OFF) { +// ignoreZkOffsets = false; +// startOffsetTime = kafka.api.OffsetRequest.LatestTime(); +// } +// return this; +// } +//} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java index 5aec998a62..8559b720a5 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java @@ -19,6 +19,6 @@ package org.apache.metron.common.spout.kafka; public interface SpoutConfigFunction { - void configure(org.apache.storm.kafka.SpoutConfig config, Object val); + void configure(org.apache.storm.kafka.spout.KafkaSpoutConfig config, Object val); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java index 7d1c7c331f..fa60973b72 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java @@ -20,7 +20,6 @@ import com.google.common.base.Joiner; import org.apache.metron.common.utils.ConversionUtils; -import org.apache.storm.kafka.SpoutConfig; import java.util.EnumMap; import java.util.Map; @@ -44,11 +43,11 @@ public enum SpoutConfigOptions implements SpoutConfigFunction { } @Override - public void configure(SpoutConfig config, Object val) { + public void configure(KafkaSpoutConfig config, Object val) { spoutConfigFunc.configure(config, val); } - public static SpoutConfig configure(SpoutConfig config, EnumMap configs) { + public static KafkaSpoutConfig configure(KafkaSpoutConfig config, EnumMap configs) { if(configs != null) { for(Map.Entry kv : configs.entrySet()) { kv.getKey().configure(config, kv.getValue()); diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 4a3ce7afe3..82c6f39387 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -106,7 +106,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index b2c3a8c797..5a27f04721 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -121,7 +121,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 14fe5c4863..2a9b8b7411 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -76,7 +76,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} test @@ -86,17 +86,6 @@ - - org.apache.kafka - kafka_2.9.2 - ${global_kafka_version} - - - log4j - log4j - - - org.apache.hadoop @@ -146,6 +135,13 @@ metron-test-utilities ${project.parent.version} + + + info.batey.kafka + kafka-unit + + 0.6 + diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java index 8ffc5e7a96..330cf60a9e 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java @@ -20,6 +20,7 @@ import com.google.common.base.Function; import kafka.admin.AdminUtils; +import kafka.admin.RackAwareMode; import kafka.api.FetchRequest; import kafka.api.FetchRequestBuilder; import kafka.common.TopicExistsException; @@ -38,7 +39,15 @@ import kafka.zk.EmbeddedZookeeper; import org.I0Itec.zkclient.ZkClient; import org.apache.metron.integration.InMemoryComponent; +import org.apache.metron.integration.wrapper.AdminUtilsWrapper; +import org.apache.metron.integration.wrapper.TestUtilsWrapper; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.*; @@ -61,6 +70,9 @@ public Topic(String name, int numPartitions) { private transient ZkClient zkClient; private transient ConsumerConnector consumer; private String zookeeperConnectString; +// def zkPort: Int = zookeeper.port +// def zkConnect: String = s + private int brokerPort = 6667; private List topics = Collections.emptyList(); private Function postStartCallback; @@ -78,8 +90,9 @@ public KafkaWithZKComponent withExistingZookeeper(String zookeeperConnectString) public KafkaWithZKComponent withBrokerPort(int brokerPort) { if(brokerPort <= 0) { - brokerPort = TestUtils.choosePort(); + brokerPort = TestUtils.RandomPort(); } + this.brokerPort = brokerPort; return this; } @@ -129,14 +142,17 @@ public KafkaProducer createProducer(Map properties, C public void start() { // setup Zookeeper if(zookeeperConnectString == null) { - String zkConnect = TestZKUtils.zookeeperConnect(); - zkServer = new EmbeddedZookeeper(zkConnect); - zookeeperConnectString = zkServer.connectString(); +// String zkConnect = TestZKUtils.zookeeperConnect(); +// zkServer = new EmbeddedZookeeper(zkConnect); +// zookeeperConnectString = zkServer.connectString(); + EmbeddedZookeeper ezk = new EmbeddedZookeeper(); + zookeeperConnectString = "127.0.0.1:" + ezk.port(); } zkClient = new ZkClient(zookeeperConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); // setup Broker - Properties props = TestUtils.createBrokerConfig(0, brokerPort, true); +// Properties props = TestUtilsWrapper.createBrokerConfig(0, brokerPort, true); + Properties props = TestUtilsWrapper.createBrokerConfig(0, zookeeperConnectString, brokerPort); props.setProperty("zookeeper.connection.timeout.ms","1000000"); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); @@ -151,6 +167,39 @@ public void start() { postStartCallback.apply(this); } +// private int port; + +// private ServerCnxnFactory factory; + +// public String startupZookeeper(String ) { +// +// File snapshotDir; +// File logDir; +// try { +// snapshotDir = java.nio.file.Files.createTempDirectory("zookeeper-snapshot").toFile(); +// logDir = java.nio.file.Files.createTempDirectory("zookeeper-logs").toFile(); +// } catch (IOException e) { +// throw new RuntimeException("Unable to start Kafka", e); +// } +// +// snapshotDir.deleteOnExit(); +// logDir.deleteOnExit(); +// +// try { +// int tickTime = 500; +// ZooKeeperServer zkServer = new ZooKeeperServer(snapshotDir, logDir, tickTime); +// this.factory = NIOServerCnxnFactory.createFactory(); +// this.factory.configure(new InetSocketAddress("localhost", port), 16); +// factory.startup(zkServer); +// } catch (InterruptedException e) { +// Thread.currentThread().interrupt(); +// } catch (IOException e) { +// throw new RuntimeException("Unable to start ZooKeeper", e); +// } +// +// return +// } + public String getZookeeperConnect() { return zookeeperConnectString; } @@ -166,6 +215,9 @@ public void stop() { if(zkServer != null) { zkServer.shutdown(); } +// if(factory != null) { +// factory.shutdown(); +// } } public List readMessages(String topic) { @@ -192,7 +244,8 @@ public ConsumerIterator getStreamIterator(String topic) { } public ConsumerIterator getStreamIterator(String topic, String group, String consumerName) { // setup simple consumer - Properties consumerProperties = TestUtils.createConsumerProperties(zkServer.connectString(), group, consumerName, -1); +// Properties consumerProperties = TestUtils.createConsumerProperties(zkServer.connectString(), group, consumerName, -1); + Properties consumerProperties = TestUtils.createConsumerProperties(zookeeperConnectString, group, consumerName, -1); consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(consumerProperties)); Map topicCountMap = new HashMap(); topicCountMap.put(topic, 1); @@ -220,7 +273,9 @@ public void waitUntilMetadataIsPropagated(String topic, int numPartitions) { public void createTopic(String name, int numPartitions, boolean waitUntilMetadataIsPropagated) throws InterruptedException { try { - AdminUtils.createTopic(zkClient, name, numPartitions, 1, new Properties()); + ZkUtils zkUtils = ZkUtils.apply(zookeeperConnectString, 30000, 30000, false); + AdminUtilsWrapper.createTopic(zkUtils, name, numPartitions, 1, new Properties()); +// AdminUtils.createTopic(zkClient, name, numPartitions, 1, new Properties()); if (waitUntilMetadataIsPropagated) { waitUntilMetadataIsPropagated(name, numPartitions); } diff --git a/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/AdminUtilsWrapper.scala b/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/AdminUtilsWrapper.scala new file mode 100644 index 0000000000..aed9dda0f3 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/AdminUtilsWrapper.scala @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.integration.wrapper + +import java.util.Properties + +import kafka.admin.AdminUtils +import kafka.utils.ZkUtils + +object AdminUtilsWrapper { + def createTopic(zkUtils: ZkUtils, + topic: String, + partitions: Int, + replicationFactor: Int, + topicConfig: Properties = new Properties) { + AdminUtils.createTopic(zkUtils, topic, partitions, replicationFactor, topicConfig) + } +} diff --git a/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/TestUtilsWrapper.scala b/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/TestUtilsWrapper.scala new file mode 100644 index 0000000000..576c8a2bd2 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/scala/org/apache/metron/integration/wrapper/TestUtilsWrapper.scala @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.integration.wrapper + +import java.util.Properties + +import kafka.utils.TestUtils + +object TestUtilsWrapper { + // Don't expose properties with defaults + def createBrokerConfig(nodeId: Int, zkConnect: String, port: Int): Properties = { + TestUtils.createBrokerConfig(nodeId = nodeId, zkConnect = zkConnect, port = port) + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index 25fbe8b728..423e21b1ff 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -139,7 +139,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} diff --git a/metron-platform/metron-parsers/src/test/resources/log4j.properties b/metron-platform/metron-parsers/src/test/resources/log4j.properties new file mode 100644 index 0000000000..d5d33a1dd9 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/log4j.properties @@ -0,0 +1,28 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Root logger option +log4j.rootLogger=WARN, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index dc3069f866..5a1017bbb9 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -45,7 +45,7 @@ org.apache.storm - storm-kafka + storm-kafka-client ${global_storm_version} @@ -89,7 +89,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml index 6b1471249c..afc0cabef2 100644 --- a/metron-platform/metron-pcap/pom.xml +++ b/metron-platform/metron-pcap/pom.xml @@ -85,7 +85,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} provided @@ -105,7 +105,7 @@ org.apache.storm - storm-kafka + storm-kafka-client ${global_storm_version} diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml index 65077e0b5f..62f726479f 100644 --- a/metron-platform/metron-solr/pom.xml +++ b/metron-platform/metron-solr/pom.xml @@ -112,7 +112,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} diff --git a/metron-platform/metron-test-utilities/pom.xml b/metron-platform/metron-test-utilities/pom.xml index 4796667615..f6015c44cd 100644 --- a/metron-platform/metron-test-utilities/pom.xml +++ b/metron-platform/metron-test-utilities/pom.xml @@ -69,7 +69,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml index 1859c7f94d..15b169854c 100644 --- a/metron-platform/metron-writer/pom.xml +++ b/metron-platform/metron-writer/pom.xml @@ -74,7 +74,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} provided From ec157af245cfcac3a8eda390d145328d62dfb0ab Mon Sep 17 00:00:00 2001 From: justinjleet Date: Thu, 13 Oct 2016 15:56:43 -0400 Subject: [PATCH 4/7] Missing changes from last commit --- metron-analytics/metron-profiler/pom.xml | 2 +- .../metron/profiler/integration/ProfilerIntegrationTest.java | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index 6f12d16199..9e308982cd 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -84,7 +84,7 @@ org.apache.kafka - kafka_2.9.2 + kafka_2.10 ${global_kafka_version} diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java index 04a997102d..5fabb68af3 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java @@ -98,7 +98,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { private ColumnBuilder columnBuilder; private FluxTopologyComponent fluxComponent; - private KafkaWithZKComponent kafkaComponent; +// private KafkaWithZKComponent kafkaComponent; private List input; private ComponentRunner runner; private MockHTable profilerTable; diff --git a/pom.xml b/pom.xml index 29c103e3de..646c05e4de 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 1.0.1 1.0.1 1.7.1 - 0.8.2.2 + 0.10.0.1 2.7.1 1.1.1 1.5.2 From 90321655055e9a290f64724c8e3bd8a013deebfc Mon Sep 17 00:00:00 2001 From: justinjleet Date: Fri, 14 Oct 2016 08:51:17 -0400 Subject: [PATCH 5/7] WIP changes with scala wrapper --- metron-platform/metron-common/pom.xml | 2 +- .../common/spout/kafka/SpoutConfig.java | 118 +++++++++--------- .../spout/kafka/SpoutConfigFunction.java | 2 +- .../spout/kafka/SpoutConfigOptions.java | 5 +- .../metron-integration-test/pom.xml | 24 ++++ metron-platform/metron-pcap-backend/pom.xml | 2 +- metron-platform/metron-pcap/pom.xml | 2 +- 7 files changed, 90 insertions(+), 65 deletions(-) diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index 581766722d..9faa256b97 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -193,7 +193,7 @@ org.apache.storm - storm-kafka-client + storm-kafka ${global_storm_version} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java index ad18813a4c..c26d1f5632 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java @@ -1,59 +1,59 @@ -///** -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package org.apache.metron.common.spout.kafka; -// -//import org.apache.storm.kafka.BrokerHosts; -// -//public class SpoutConfig extends org.apache.storm.kafka.spout.KafkaSpoutConfig { -// public static enum Offset { -// BEGINNING, END, WHERE_I_LEFT_OFF; -// } -// public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) { -// super(hosts, topic, zkRoot, id); -// } -// -// public SpoutConfig from(String offset) { -// try { -// Offset o = Offset.valueOf(offset); -// from(o); -// } -// catch(IllegalArgumentException iae) { -// from(Offset.WHERE_I_LEFT_OFF); -// } -// ignoreZkOffsets = true; -// startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); -// return this; -// } -// -// public SpoutConfig from(Offset offset) { -// if(offset == Offset.BEGINNING) { -// ignoreZkOffsets = true; -// startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); -// } -// else if(offset == Offset.END) { -// ignoreZkOffsets = true; -// startOffsetTime = kafka.api.OffsetRequest.LatestTime(); -// } -// else if(offset == Offset.WHERE_I_LEFT_OFF) { -// ignoreZkOffsets = false; -// startOffsetTime = kafka.api.OffsetRequest.LatestTime(); -// } -// return this; -// } -//} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.spout.kafka; + +import org.apache.storm.kafka.BrokerHosts; + +public class SpoutConfig extends org.apache.storm.kafka.SpoutConfig { + public static enum Offset { + BEGINNING, END, WHERE_I_LEFT_OFF; + } + public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) { + super(hosts, topic, zkRoot, id); + } + + public SpoutConfig from(String offset) { + try { + Offset o = Offset.valueOf(offset); + from(o); + } + catch(IllegalArgumentException iae) { + from(Offset.WHERE_I_LEFT_OFF); + } + ignoreZkOffsets = true; + startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); + return this; + } + + public SpoutConfig from(Offset offset) { + if(offset == Offset.BEGINNING) { + ignoreZkOffsets = true; + startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); + } + else if(offset == Offset.END) { + ignoreZkOffsets = true; + startOffsetTime = kafka.api.OffsetRequest.LatestTime(); + } + else if(offset == Offset.WHERE_I_LEFT_OFF) { + ignoreZkOffsets = false; + startOffsetTime = kafka.api.OffsetRequest.LatestTime(); + } + return this; + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java index 8559b720a5..5aec998a62 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigFunction.java @@ -19,6 +19,6 @@ package org.apache.metron.common.spout.kafka; public interface SpoutConfigFunction { - void configure(org.apache.storm.kafka.spout.KafkaSpoutConfig config, Object val); + void configure(org.apache.storm.kafka.SpoutConfig config, Object val); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java index fa60973b72..7d1c7c331f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfigOptions.java @@ -20,6 +20,7 @@ import com.google.common.base.Joiner; import org.apache.metron.common.utils.ConversionUtils; +import org.apache.storm.kafka.SpoutConfig; import java.util.EnumMap; import java.util.Map; @@ -43,11 +44,11 @@ public enum SpoutConfigOptions implements SpoutConfigFunction { } @Override - public void configure(KafkaSpoutConfig config, Object val) { + public void configure(SpoutConfig config, Object val) { spoutConfigFunc.configure(config, val); } - public static KafkaSpoutConfig configure(KafkaSpoutConfig config, EnumMap configs) { + public static SpoutConfig configure(SpoutConfig config, EnumMap configs) { if(configs != null) { for(Map.Entry kv : configs.entrySet()) { kv.getKey().configure(config, kv.getValue()); diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 2a9b8b7411..6f7a8dfcf6 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -144,6 +144,30 @@ + + + net.alchim31.maven + scala-maven-plugin + 3.2.1 + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index 5a1017bbb9..ddbbb4989e 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -45,7 +45,7 @@ org.apache.storm - storm-kafka-client + storm-kafka ${global_storm_version} diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml index afc0cabef2..4f0ce71e2a 100644 --- a/metron-platform/metron-pcap/pom.xml +++ b/metron-platform/metron-pcap/pom.xml @@ -105,7 +105,7 @@ org.apache.storm - storm-kafka-client + storm-kafka ${global_storm_version} From f418f50958d9826c9307858914253f304a18d066 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Fri, 14 Oct 2016 12:22:59 -0400 Subject: [PATCH 6/7] More integration tests working --- dependencies_with_url.csv | 2 +- .../src/main/resources/META-INF/LICENSE | 2 +- metron-analytics/metron-profiler/pom.xml | 13 +++++++ .../src/main/flux/profiler/remote.yaml | 4 +-- .../src/main/resources/META-INF/LICENSE | 2 +- .../integration/ProfilerIntegrationTest.java | 2 +- .../roles/ambari_config/defaults/main.yml | 2 +- .../roles/flume/defaults/main.yml | 2 +- .../roles/kafka-broker/defaults/main.yml | 2 +- .../roles/kafka-client/defaults/main.yml | 2 +- .../src/main/resources/META-INF/LICENSE | 2 +- metron-platform/metron-elasticsearch/pom.xml | 13 +++++++ .../src/main/resources/META-INF/LICENSE | 2 +- metron-platform/metron-enrichment/pom.xml | 13 +++++++ .../src/main/flux/enrichment/remote.yaml | 6 ++-- .../src/main/flux/enrichment/test.yaml | 6 ++-- .../src/main/resources/META-INF/LICENSE | 2 +- .../src/main/flux/indexing/remote.yaml | 4 +-- .../components/KafkaWithZKComponent.java | 36 ------------------- metron-platform/metron-parsers/pom.xml | 13 +++++++ .../src/main/resources/META-INF/LICENSE | 2 +- metron-platform/metron-pcap-backend/pom.xml | 35 ++++++++++++++++-- .../src/main/flux/pcap/remote.yaml | 2 +- .../metron/spout/pcap/HDFSWriterCallback.java | 3 ++ .../src/main/resources/META-INF/LICENSE | 2 +- .../storm/kafka/CallbackKafkaSpout.java | 8 ++--- metron-platform/metron-solr/pom.xml | 13 +++++++ 27 files changed, 128 insertions(+), 67 deletions(-) diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index b1a2baced8..616f9cbe34 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -49,7 +49,7 @@ org.jruby.joni:joni:jar:2.1.2:compile,MIT License,https://github.com/jruby/joni org.mitre.taxii:taxii:jar:1.1.0.1:compile,The BSD 3-Clause License,https://github.com/TAXIIProject/java-taxii org.mitre:stix:jar:1.2.0.2:compile,The BSD 3-Clause License,https://github.com/STIXProject/java-stix org.mockito:mockito-core:jar:1.10.19:compile,The MIT License,http://www.mockito.org -org.scala-lang:scala-library:jar:2.9.2:compile,BSD-like,http://www.scala-lang.org/ +org.scala-lang:scala-library:jar:2.10.6:compile,BSD-like,http://www.scala-lang.org/ xmlenc:xmlenc:jar:0.52:compile,The BSD License,http://xmlenc.sourceforge.net asm:asm:jar:3.1:compile,BSD,http://asm.ow2.org/ com.sun.jersey.contribs:jersey-guice:jar:1.9:compile,CDDL 1.1,https://jersey.java.net/ diff --git a/metron-analytics/metron-profiler-common/src/main/resources/META-INF/LICENSE b/metron-analytics/metron-profiler-common/src/main/resources/META-INF/LICENSE index e5616e782d..cc1764e362 100644 --- a/metron-analytics/metron-profiler-common/src/main/resources/META-INF/LICENSE +++ b/metron-analytics/metron-profiler-common/src/main/resources/META-INF/LICENSE @@ -228,7 +228,7 @@ This product bundles leveldbjni-all 1.8, which is available under a "BSD Softwar This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni This product bundles asm 4.0, which is available under a "BSD Software License" license. For details, see http://asm.ow2.org/ -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index 9e308982cd..e67852970e 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -93,6 +93,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + org.mockito mockito-all diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml index db9739281c..b1de56528f 100644 --- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml +++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml @@ -50,7 +50,7 @@ components: ref: "defaultExecutor" - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" @@ -73,7 +73,7 @@ components: spouts: - id: "kafkaSpout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "kafkaConfig" diff --git a/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE b/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE index e5616e782d..cc1764e362 100644 --- a/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE +++ b/metron-analytics/metron-profiler/src/main/resources/META-INF/LICENSE @@ -228,7 +228,7 @@ This product bundles leveldbjni-all 1.8, which is available under a "BSD Softwar This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni This product bundles asm 4.0, which is available under a "BSD Software License" license. For details, see http://asm.ow2.org/ -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java index 5fabb68af3..04a997102d 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java @@ -98,7 +98,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { private ColumnBuilder columnBuilder; private FluxTopologyComponent fluxComponent; -// private KafkaWithZKComponent kafkaComponent; + private KafkaWithZKComponent kafkaComponent; private List input; private ComponentRunner runner; private MockHTable profilerTable; diff --git a/metron-deployment/roles/ambari_config/defaults/main.yml b/metron-deployment/roles/ambari_config/defaults/main.yml index 0badef1f89..9ac0342c46 100644 --- a/metron-deployment/roles/ambari_config/defaults/main.yml +++ b/metron-deployment/roles/ambari_config/defaults/main.yml @@ -34,4 +34,4 @@ mapred_reduce_java_opts : -Xmx1024m mapred_map_mem_mb : 1229 mapred_reduce_mem_mb : 1229 topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf' -hdp_stack: "2.4" +hdp_stack: "2.5" diff --git a/metron-deployment/roles/flume/defaults/main.yml b/metron-deployment/roles/flume/defaults/main.yml index 38ed75521f..d96c844cac 100644 --- a/metron-deployment/roles/flume/defaults/main.yml +++ b/metron-deployment/roles/flume/defaults/main.yml @@ -15,5 +15,5 @@ # limitations under the License. # --- -hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.2.0/hdp.repo" +hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo" java_home: /usr/jdk64/jdk1.8.0_60 diff --git a/metron-deployment/roles/kafka-broker/defaults/main.yml b/metron-deployment/roles/kafka-broker/defaults/main.yml index 8e22e9fd0a..b0f5a11400 100644 --- a/metron-deployment/roles/kafka-broker/defaults/main.yml +++ b/metron-deployment/roles/kafka-broker/defaults/main.yml @@ -15,4 +15,4 @@ # limitations under the License. # --- -hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.2.0/hdp.repo" +hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo" diff --git a/metron-deployment/roles/kafka-client/defaults/main.yml b/metron-deployment/roles/kafka-client/defaults/main.yml index 8e22e9fd0a..b0f5a11400 100644 --- a/metron-deployment/roles/kafka-client/defaults/main.yml +++ b/metron-deployment/roles/kafka-client/defaults/main.yml @@ -15,4 +15,4 @@ # limitations under the License. # --- -hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.2.0/hdp.repo" +hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo" diff --git a/metron-platform/metron-data-management/src/main/resources/META-INF/LICENSE b/metron-platform/metron-data-management/src/main/resources/META-INF/LICENSE index 19827c3550..52f4ee084c 100644 --- a/metron-platform/metron-data-management/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-data-management/src/main/resources/META-INF/LICENSE @@ -221,6 +221,6 @@ This product bundles jcodings 1.0.8, which is available under a "MIT Software Li This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni This product bundles taxii 1.1.0.1, which is available under a "BSD Software License" license. For details, see https://github.com/TAXIIProject/java-taxii This product bundles stix 1.2.0.2, which is available under a "BSD Software License" license. For details, see https://github.com/STIXProject/java-stix -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.5, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.5, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 82c6f39387..6bb0979a8e 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -115,6 +115,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + io.netty netty diff --git a/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE b/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE index 57abbc5d18..b3013a0668 100644 --- a/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE @@ -219,6 +219,6 @@ This product bundles jopt-simple 3.2, which is available under a "MIT Software L This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.7, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index 5a27f04721..773c6b303d 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -130,6 +130,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + org.apache.hadoop hadoop-common diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml index e182adebdb..03f48a0803 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml @@ -173,11 +173,11 @@ components: #kafka/zookeeper - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" - id: "kafkaConfig" - className: "storm.kafka.SpoutConfig" + className: "org.apache.storm.kafka.SpoutConfig" constructorArgs: # zookeeper hosts - ref: "zkHosts" @@ -195,7 +195,7 @@ components: spouts: - id: "kafkaSpout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "kafkaConfig" bolts: diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml index f8d7b83139..ca5868f46f 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml @@ -153,11 +153,11 @@ components: #kafka/zookeeper - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" - id: "kafkaConfig" - className: "storm.kafka.SpoutConfig" + className: "org.apache.storm.kafka.SpoutConfig" constructorArgs: # zookeeper hosts - ref: "zkHosts" @@ -175,7 +175,7 @@ components: spouts: - id: "kafkaSpout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "kafkaConfig" bolts: diff --git a/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE b/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE index 4858b45ed1..5819b8c5ce 100644 --- a/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE @@ -219,5 +219,5 @@ This product bundles jopt-simple 3.2, which is available under a "MIT Software L This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml index d1e697bb4c..8bf8f48a40 100644 --- a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml +++ b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml @@ -64,7 +64,7 @@ components: #kafka/zookeeper - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" - id: "kafkaConfig" @@ -85,7 +85,7 @@ components: spouts: - id: "kafkaSpout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "kafkaConfig" bolts: diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java index 330cf60a9e..20ce721cf6 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java @@ -167,39 +167,6 @@ public void start() { postStartCallback.apply(this); } -// private int port; - -// private ServerCnxnFactory factory; - -// public String startupZookeeper(String ) { -// -// File snapshotDir; -// File logDir; -// try { -// snapshotDir = java.nio.file.Files.createTempDirectory("zookeeper-snapshot").toFile(); -// logDir = java.nio.file.Files.createTempDirectory("zookeeper-logs").toFile(); -// } catch (IOException e) { -// throw new RuntimeException("Unable to start Kafka", e); -// } -// -// snapshotDir.deleteOnExit(); -// logDir.deleteOnExit(); -// -// try { -// int tickTime = 500; -// ZooKeeperServer zkServer = new ZooKeeperServer(snapshotDir, logDir, tickTime); -// this.factory = NIOServerCnxnFactory.createFactory(); -// this.factory.configure(new InetSocketAddress("localhost", port), 16); -// factory.startup(zkServer); -// } catch (InterruptedException e) { -// Thread.currentThread().interrupt(); -// } catch (IOException e) { -// throw new RuntimeException("Unable to start ZooKeeper", e); -// } -// -// return -// } - public String getZookeeperConnect() { return zookeeperConnectString; } @@ -215,9 +182,6 @@ public void stop() { if(zkServer != null) { zkServer.shutdown(); } -// if(factory != null) { -// factory.shutdown(); -// } } public List readMessages(String topic) { diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index 423e21b1ff..3e42b35c46 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -148,6 +148,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + commons-cli commons-cli diff --git a/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE b/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE index 7b3f8d0b02..0970a173b2 100644 --- a/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE @@ -219,7 +219,7 @@ This product bundles jopt-simple 3.2, which is available under a "MIT Software L This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index ddbbb4989e..79063e12e4 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -49,11 +49,18 @@ ${global_storm_version} - org.apache.curator - curator-client + org.apache.curator + curator-client + + org.apache.curator + curator-client + 2.10.0 + test + + junit junit @@ -98,6 +105,30 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + + + log4j + log4j + + + + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + org.apache.storm storm-core diff --git a/metron-platform/metron-pcap-backend/src/main/flux/pcap/remote.yaml b/metron-platform/metron-pcap-backend/src/main/flux/pcap/remote.yaml index 32f994f67a..272ac1305b 100644 --- a/metron-platform/metron-pcap-backend/src/main/flux/pcap/remote.yaml +++ b/metron-platform/metron-pcap-backend/src/main/flux/pcap/remote.yaml @@ -20,7 +20,7 @@ config: components: - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zk}" - id: "kafkaConfig" diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java index d9f6831995..ee3ba741a9 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java @@ -132,6 +132,9 @@ private PartitionHDFSWriter getWriter(Partition partition) { @Override public void initialize(EmitContext context) { + System.out.println("***** EMITCONTEXT - TOPIC " + context.get(EmitContext.Type.TOPIC)); + System.out.println("***** EMITCONTEXT - UUID " + context.get(EmitContext.Type.UUID)); + System.out.println("***** EMITCONTEXT - SPOUTCONFIG " + context.get(EmitContext.Type.SPOUT_CONFIG)); this.context = context; this.topic = context.get(EmitContext.Type.TOPIC); } diff --git a/metron-platform/metron-pcap-backend/src/main/resources/META-INF/LICENSE b/metron-platform/metron-pcap-backend/src/main/resources/META-INF/LICENSE index 0905df5b28..ed6a2522bf 100644 --- a/metron-platform/metron-pcap-backend/src/main/resources/META-INF/LICENSE +++ b/metron-platform/metron-pcap-backend/src/main/resources/META-INF/LICENSE @@ -219,7 +219,7 @@ This product bundles jaxb-api 2.2.2, which is available under a "Common Developm This product bundles stax-api 1.0-2, which is available under a "Common Development and Distribution License v1.0" license. For details, see https://docs.oracle.com/javase/7/docs/api/javax/xml/stream/package-summary.html This product bundles jopt-simple 3.2, which is available under a "MIT Software License" license. For details, see http://jopt-simple.sourceforge.net This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni -This product bundles scala-library 2.9.2, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ This product bundles slf4j-api 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java index b7bcb58d5a..661fb0b94d 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/storm/kafka/CallbackKafkaSpout.java @@ -38,11 +38,9 @@ public CallbackKafkaSpout(SpoutConfig spoutConf, Class callb public void initialize(TopologyContext context) { _callback = createCallback(callbackClazz); - //TODO figure out if there's a better way to handle this now -// _context = new EmitContext().with(EmitContext.Type.SPOUT_CONFIG, _spoutConfig) -// .with(EmitContext.Type.UUID, context.getStormId()) -// .with(EmitContext.Type.TOPIC, _spoutConfig.topic); - _context = new EmitContext().with(EmitContext.Type.TOPOLOGY_CONTEXT, context); + _context = new EmitContext().with(EmitContext.Type.SPOUT_CONFIG, _spoutConfig) + .with(EmitContext.Type.UUID, context.getStormId()) + .with(EmitContext.Type.TOPIC, _spoutConfig.topic); _callback.initialize(_context); } diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml index 62f726479f..d87e860651 100644 --- a/metron-platform/metron-solr/pom.xml +++ b/metron-platform/metron-solr/pom.xml @@ -121,6 +121,19 @@ + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + org.apache.metron metron-integration-test From f445bfb9fef0583bbce444406227f96a4c506706 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Fri, 14 Oct 2016 15:24:41 -0400 Subject: [PATCH 7/7] cleanup --- metron-platform/metron-data-management/pom.xml | 8 ++++++++ metron-platform/metron-integration-test/pom.xml | 7 ------- metron-platform/metron-pcap-backend/pom.xml | 12 ++++-------- .../apache/metron/spout/pcap/HDFSWriterCallback.java | 3 --- .../integration/PcapTopologyIntegrationTest.java | 2 +- 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml index 0263396ef0..f39139dc23 100644 --- a/metron-platform/metron-data-management/pom.xml +++ b/metron-platform/metron-data-management/pom.xml @@ -208,6 +208,14 @@ org.apache.hadoop hadoop-mapreduce-client-common + + log4j + log4j + + + log4j-over-slf4j + org.slf4j + diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 6f7a8dfcf6..2dbd87e147 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -135,13 +135,6 @@ metron-test-utilities ${project.parent.version} - - - info.batey.kafka - kafka-unit - - 0.6 - diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index 79063e12e4..c8728570a4 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -49,17 +49,13 @@ ${global_storm_version} - org.apache.curator - curator-client + + + curator-client + org.apache.curator - - org.apache.curator - curator-client - 2.10.0 - test - junit diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java index ee3ba741a9..d9f6831995 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java @@ -132,9 +132,6 @@ private PartitionHDFSWriter getWriter(Partition partition) { @Override public void initialize(EmitContext context) { - System.out.println("***** EMITCONTEXT - TOPIC " + context.get(EmitContext.Type.TOPIC)); - System.out.println("***** EMITCONTEXT - UUID " + context.get(EmitContext.Type.UUID)); - System.out.println("***** EMITCONTEXT - SPOUTCONFIG " + context.get(EmitContext.Type.SPOUT_CONFIG)); this.context = context; this.topic = context.get(EmitContext.Type.TOPIC); } diff --git a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java index 77b5f9db69..fcdb108d79 100644 --- a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java +++ b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java @@ -240,7 +240,7 @@ public Void apply(@Nullable KafkaWithZKComponent kafkaWithZKComponent) { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("mr", mr) .withComponent("kafka", kafkaComponent) - .withComponent("org/apache/storm", fluxComponent) + .withComponent("storm", fluxComponent) .withMaxTimeMS(-1) .withMillisecondsBetweenAttempts(2000) .withNumRetries(10)