From 263c6fd498a1f28846892b6fd9fbcc8c52629fd5 Mon Sep 17 00:00:00 2001 From: Sean Schulte Date: Wed, 6 Jan 2016 14:45:08 -0600 Subject: [PATCH 1/7] Add Flux to POM. --- opensoc-streaming/OpenSOC-Topologies/pom.xml | 5 +++++ .../java/com/opensoc/topology/runner/TopologyRunner.java | 1 - opensoc-streaming/pom.xml | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/opensoc-streaming/OpenSOC-Topologies/pom.xml b/opensoc-streaming/OpenSOC-Topologies/pom.xml index 3ec016f0c2..d01118337e 100644 --- a/opensoc-streaming/OpenSOC-Topologies/pom.xml +++ b/opensoc-streaming/OpenSOC-Topologies/pom.xml @@ -104,6 +104,11 @@ storm-kafka ${global_storm_version} + + org.apache.storm + flux-core + ${global_flux_version} + org.apache.storm storm diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/TopologyRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/TopologyRunner.java index 72c2240e0a..ed3282539f 100644 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/TopologyRunner.java +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/TopologyRunner.java @@ -555,7 +555,6 @@ private boolean initializeKafkaSpout(String name) { SpoutConfig kafkaConfig = new SpoutConfig(zk, input_topic, "", input_topic); kafkaConfig.scheme = new SchemeAsMultiScheme(new RawScheme()); - kafkaConfig.forceFromStart = Boolean.valueOf("True"); kafkaConfig.startOffsetTime = -1; builder.setSpout(name, new KafkaSpout(kafkaConfig), diff --git a/opensoc-streaming/pom.xml b/opensoc-streaming/pom.xml index bbd4e2e6ec..66aac8a8e5 100644 --- a/opensoc-streaming/pom.xml +++ b/opensoc-streaming/pom.xml @@ -21,7 +21,8 @@ www.getopensoc.com @ProjectOpenSOC - 0.9.2-incubating + 0.10.0 + 0.10.0 0.8.0 2.2.0 0.98.0-hadoop2 From ffe0369925d90f0a261f1ca902b5bd90ce4b3e10 Mon Sep 17 00:00:00 2001 From: Sean Schulte Date: Fri, 8 Jan 2016 14:27:03 -0600 Subject: [PATCH 2/7] Started local Bro topology with Flux. --- .../test/spouts/GenericInternalTestSpout.java | 6 +- .../OpenSOC_Configs/etc/env/config.properties | 62 ++++++ .../OpenSOC_Configs/topologies/bro/local.yaml | 185 ++++++++++++++++++ 3 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/config.properties create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/local.yaml diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/test/spouts/GenericInternalTestSpout.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/test/spouts/GenericInternalTestSpout.java index ced5266267..ccb54ab9e8 100644 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/test/spouts/GenericInternalTestSpout.java +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/test/spouts/GenericInternalTestSpout.java @@ -55,13 +55,13 @@ public GenericInternalTestSpout withFilename(String filename) _filename = filename; return this; } - public GenericInternalTestSpout withMilisecondDelay(int delay) + public GenericInternalTestSpout withMillisecondDelay(Integer delay) { _delay = delay; return this; } - public GenericInternalTestSpout withRepeating(boolean repeating) + public GenericInternalTestSpout withRepeating(Boolean repeating) { _repeating = repeating; return this; @@ -111,4 +111,4 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("message")); } -} \ No newline at end of file +} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/config.properties b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/config.properties new file mode 100644 index 0000000000..bd03821144 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/config.properties @@ -0,0 +1,62 @@ +##### Kafka ##### + +kafka.zk=zkpr1:2181,zkpr2:2181,zkpr3:2181 +spout.kafka.topic=bro_raw + +##### ElasticSearch ##### + +es.ip=10.22.0.214 +es.port=9300 +es.clustername=elasticsearch + +##### MySQL ##### + +mysql.ip=10.22.0.214 +mysql.port=3306 +mysql.username=root +mysql.password=hadoop123 + +##### Metrics ##### + +#reporters +com.opensoc.metrics.reporter.graphite=true +com.opensoc.metrics.reporter.console=false +com.opensoc.metrics.reporter.jmx=false + +#Graphite Addresses + +com.opensoc.metrics.graphite.address=localhost +com.opensoc.metrics.graphite.port=2023 + +#TelemetryParserBolt +com.opensoc.metrics.TelemetryParserBolt.acks=true +com.opensoc.metrics.TelemetryParserBolt.emits=true +com.opensoc.metrics.TelemetryParserBolt.fails=true + + +#GenericEnrichmentBolt +com.opensoc.metrics.GenericEnrichmentBolt.acks=true +com.opensoc.metrics.GenericEnrichmentBolt.emits=true +com.opensoc.metrics.GenericEnrichmentBolt.fails=true + + +#TelemetryIndexingBolt +com.opensoc.metrics.TelemetryIndexingBolt.acks=true +com.opensoc.metrics.TelemetryIndexingBolt.emits=true +com.opensoc.metrics.TelemetryIndexingBolt.fails=true + +##### Host Enrichment ##### + +com.opensoc.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ +{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ +{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] + +##### HDFS ##### + +bolt.hdfs.batch.size=5000 +bolt.hdfs.field.delimiter=| +bolt.hdfs.file.rotation.size.in.mb=5 +bolt.hdfs.file.system.url=hdfs://iot01.cloud.hortonworks.com:8020 +bolt.hdfs.wip.file.path=/paloalto/wip +bolt.hdfs.finished.file.path=/paloalto/rotated +bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/local.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/local.yaml new file mode 100644 index 0000000000..2c836503a7 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/local.yaml @@ -0,0 +1,185 @@ +name: "bro-local" +config: + topology.workers: 1 + +components: + - id: "broParser" + className: "com.opensoc.parsing.parsers.BasicBroParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "com.opensoc.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/BroExampleOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "broParser" + - name: "withOutputFieldName" + args: + - "bro" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "bro_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "bro_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "bro_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> indexing" + from: "parserBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE From 971a478024d00e566359758de86ae44e4f36a807 Mon Sep 17 00:00:00 2001 From: Sean Schulte Date: Tue, 12 Jan 2016 15:34:46 -0600 Subject: [PATCH 3/7] Added remote Bro topology. --- .../OpenSOC-Topologies/readme.md | 46 ++-- .../OpenSOC_Configs/topologies/bro/alerts.xml | 24 --- .../topologies/bro/features_enabled.conf | 113 ---------- .../topologies/bro/metrics.conf | 26 --- .../topologies/bro/remote.yaml | 199 ++++++++++++++++++ .../topologies/bro/topology.conf | 137 ------------ .../topologies/bro/topology_identifier.conf | 4 - 7 files changed, 218 insertions(+), 331 deletions(-) delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/alerts.xml delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/features_enabled.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/metrics.conf create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/remote.yaml delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/topology.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/topology_identifier.conf diff --git a/opensoc-streaming/OpenSOC-Topologies/readme.md b/opensoc-streaming/OpenSOC-Topologies/readme.md index feac62da2d..306a003f2e 100644 --- a/opensoc-streaming/OpenSOC-Topologies/readme.md +++ b/opensoc-streaming/OpenSOC-Topologies/readme.md @@ -6,42 +6,34 @@ This module provides example topologies that show how to drive OpenSOC modules a ##Launching Topologies +We use Storm Flux to launch topologies, which are each described in a YAML file. ``` +storm jar target/OpenSOC-Topologies-0.6BETA.jar org.apache.storm.flux.Flux --local src/main/resources/OpenSOC_Configs/topologies/bro/local.yaml --filter src/main/resources/OpenSOC_Configs/etc/env/config.properties -storm jar OpenSOC-Topologies-0.6BETA.jar com.opensoc.topology.Pcap -storm jar OpenSOC-Topologies-0.6BETA.jar com.opensoc.topology.Sourcefire -storm jar OpenSOC-Topologies-0.6BETA.jar com.opensoc.topology.Lancope -storm jar OpenSOC-Topologies-0.6BETA.jar com.opensoc.topology.Ise - -Topology Options: --config_path OPTIONAL ARGUMENT [/path/to/configs] Path to -configuration folder. If not provided topology -will initialize with default configs --debug OPTIONAL ARGUMENT [true|false] Storm debugging -enabled. Default value is true --generator_spout REQUIRED ARGUMENT [true|false] Turn on test -generator spout. Default is set to false. If -test generator spout is turned on then kafka -spout is turned off. Instead the generator -spout will read telemetry from file and ingest -it into a topology --h Display help menue --local_mode REQUIRED ARGUMENT [true|false] Local mode or -cluster mode. If set to true the topology will -run in local mode. If set to false the topology -will be deployed to Storm nimbus +storm jar target/OpenSOC-Topologies-0.6BETA.jar org.apache.storm.flux.Flux --remote src/main/resources/OpenSOC_Configs/topologies/bro/remote.yaml --filter src/main/resources/OpenSOC_Configs/etc/env/config.properties ``` +Note that if you use `--local` it will run the topology in local mode, using test data. If you use `--remote` it will attempt to connect to and deploy to Storm Nimbus. + +Each topology's YAML files are responsible for either connecting to a real spout or enabling their own testing spout. This is the primary reason different `local.yaml` and `remote.yaml` files are provided for each topology. + ##Topology Configs The sample topologies provided use a specific directory structure. The example directory structure was checked in here: ``` -https://github.com/OpenSOC/opensoc-streaming/tree/master/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs +https://github.com/apache/incubator-metron/tree/master/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs ``` -topology.conf - settings specific to each topology -features_enabled.conf - turn on and off features for each topology and control parallelism -metrics.conf - export definitions for metrics to Graphite -topology_dentifier.conf - customer-specific tag (since we deploy to multiple data centers we need to identify where the alerts are coming from and what topologies we are looking at when we need to debug) +Each topology has a `local.yaml` and a `remote.yaml` file to support local mode and remote mode, respectively. + +These topology configurations have variables that can be replaced by the `--filter` option to Flux. These variables are in `src/main/resources/OpenSOC_Configs/etc/env/config.properties`, and apply to: + +- Kafka +- Elasticsearch +- MySQL +- Metrics +- Bolt acks/emits/fails +- Host enrichment +- HDFS diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/alerts.xml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/alerts.xml deleted file mode 100644 index 3016afbfdb..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/alerts.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - .*host\"\:\{"ip_dst_addr\"\:\{\},\"ip_src_addr\"\:\{\}.* - {"type":"error","priority":5, "title":"No Local Hostname Present", "body": - "We don't have a record for source or destination IPs in our internal database."} - - - - .*whois\"\:\{\"tld\"\:\{\}.* - {"type":"warning","priority":10, "title":"Whois domain unknown", "body": - "Could not locate whois information for tld"} - - - ^((?!country\"\:\"US\").)*$ - {"type":"warning","priority":10, "title":"NOT US IP", "body": "Communication contains a non-US IP"} - - - .*geo.* - {"type":"error","priority":1, "title":"test", "body": "test alert"} - - - - diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/features_enabled.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/features_enabled.conf deleted file mode 100644 index 5b45ddef9a..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/features_enabled.conf +++ /dev/null @@ -1,113 +0,0 @@ -#Enable and disable features for each topology - -#Feature: Test spout -##Feature Description: Reads telemetry from file and ingests it into topology. Used for testing or bulk loading the topology - -spout.test.name=TestSpout -spout.test.enabled=true -spout.test.num.tasks=1 -spout.test.parallelism.hint=1 - -#Feature: Kafka spout -##Feature Description: Acts as a Kafka consumer. Takes messages from a Kafka topic and ingests them into a topology - -spout.kafka.name=KafkaSpout -spout.kafka.enabled=false -spout.kafka.num.tasks=1 -spout.kafka.parallelism.hint=1 - -#Feature: Parser Bolt -##Feature Description: Parses telemetry from its native format into a native JSON - -bolt.parser.name=ParserBolt -bolt.parser.enabled=true -bolt.parser.num.tasks=1 -bolt.parser.parallelism.hint=1 - -#Feature: Host Enrichment -##Feature Description: Appends information about known hosts to a telemetry message - -bolt.enrichment.host.name=HostEnrichment -bolt.enrichment.host.enabled=true -bolt.enrichment.host.num.tasks=1 -bolt.enrichment.host.parallelism.hint=1 - -#Feature: Geo Enrichment -##Feature Description: Appends geo information about known non-local IPs to a telemetry message - -bolt.enrichment.geo.name=GeoEnrichment -bolt.enrichment.geo.enabled=true -bolt.enrichment.geo.num.tasks=1 -bolt.enrichment.geo.parallelism.hint=1 - -#Feature: Whois Enrichment -##Feature Description: Appends whois information about known domains to a telemetry message - -bolt.enrichment.whois.name=WhoisEnrichment -bolt.enrichment.whois.enabled=false -bolt.enrichment.whois.num.tasks=1 -bolt.enrichment.whois.parallelism.hint=1 - -#Feature: CIF Enrichment -##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message - -bolt.enrichment.cif.name=SIFBolt -bolt.enrichment.cif.enabled=false -bolt.enrichment.cif.num.tasks=1 -bolt.enrichment.cif.parallelism.hint=1 - -#Feature: Threat Enrichment -##Feature Description: Appends information from Threat intelligence feeds to a telemetry message - -bolt.enrichment.threat.name=ThreatBolt -bolt.enrichment.threat.enabled=false -bolt.enrichment.threat.num.tasks=1 -bolt.enrichment.threat.parallelism.hint=1 - -#Feature: Rules-Based Alerts -##Feature Description: Tags messages with rules-based alerts - -bolt.alerts.name=Alerts -bolt.alerts.enabled=true -bolt.alerts.num.tasks=1 -bolt.alerts.parallelism.hint=1 - -#Feature: Indexer -##Feature Description: Indexes telemetry messages in ElasticSearch or Solr - -bolt.indexing.name=IndexBolt -bolt.indexing.enabled=true -bolt.indexing.num.tasks=1 -bolt.indexing.parallelism.hint=1 - -#Feature: Alerts Indexer -##Feature Description: Indexes alert messages in ElasticSearch or Solr - -bolt.alerts.indexing.name=AlertIndexBolt -bolt.alerts.indexing.enabled=true -bolt.alerts.indexing.num.tasks=1 -bolt.alerts.indexing.parallelism.hint=1 - -#Feature: Error Indexer -##Feature Description: Indexes error messages in ElasticSearch or Solr - -bolt.error.indexing.name=ErrorIndexBolt -bolt.error.indexing.enabled=true -bolt.error.indexing.num.tasks=1 -bolt.error.indexing.parallelism.hint=1 - -#Feature: Kafka Bolt -##Feature Description: Writes telemetry messages back into a Kafka topic - -bolt.kafka.name=KafkaBolt -bolt.kafka.enabled=false -bolt.kafka.num.tasks=1 -bolt.kafka.parallelism.hint=1 - -#Feature: HDFS Bolt -##Feature Description: Writes telemetry messages into HDFS - -bolt.hdfs.name=HDFSBolt -bolt.hdfs.enabled=false -bolt.hdfs.num.tasks=1 -bolt.hdfs.parallelism.hint=1 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/metrics.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/metrics.conf deleted file mode 100644 index 1daef3d889..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/metrics.conf +++ /dev/null @@ -1,26 +0,0 @@ -#reporters -com.opensoc.metrics.reporter.graphite=true -com.opensoc.metrics.reporter.console=false -com.opensoc.metrics.reporter.jmx=false - -#Graphite Addresses - -com.opensoc.metrics.graphite.address=localhost -com.opensoc.metrics.graphite.port=2023 - -#TelemetryParserBolt -com.opensoc.metrics.TelemetryParserBolt.acks=true -com.opensoc.metrics.TelemetryParserBolt.emits=true -com.opensoc.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -com.opensoc.metrics.GenericEnrichmentBolt.acks=true -com.opensoc.metrics.GenericEnrichmentBolt.emits=true -com.opensoc.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -com.opensoc.metrics.TelemetryIndexingBolt.acks=true -com.opensoc.metrics.TelemetryIndexingBolt.emits=true -com.opensoc.metrics.TelemetryIndexingBolt.fails=true diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/remote.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/remote.yaml new file mode 100644 index 0000000000..9d0db5c06f --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/remote.yaml @@ -0,0 +1,199 @@ +name: "bro" +config: + topology.workers: 1 + +components: + - id: "broParser" + className: "com.opensoc.parsing.parsers.BasicBroParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "forceFromStart" + value: true + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "broParser" + - name: "withOutputFieldName" + args: + - "bro" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "bro_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "bro_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "bro_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> indexing" + from: "parserBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/topology.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/topology.conf deleted file mode 100644 index 0012aea6ae..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/topology.conf +++ /dev/null @@ -1,137 +0,0 @@ -include = ../../etc/env/environment_common.conf -include = ../../etc/env/es_connection.conf -include = ../../etc/env/hdfs_connection.conf -include = ../../etc/env/mysql_connection.conf -include = metrics.conf -include = features_enabled.conf - -#Global Properties - -debug.mode=true -local.mode=true -num.workers=1 - -#Standard 5-tuple fields - -source.ip=ip_src_addr -source.port=ip_src_port -dest.ip=ip_dst_addr -dest.port=ip_dst_port -protocol=protocol - -#Test Spout -spout.test.parallelism.repeat=false - -#Kafka Spout -spout.kafka.topic=bro_raw - -#Parsing Bolt -bolt.parser.adapter=com.opensoc.parsing.parsers.BasicBroParser -source.include.protocols=snmp,http,ftp,ssh,ssl,dns,socks,dnp3,smtp,dhcp,modbus,radius,irc -source.exclude.protocols=x509,files,app_stats - -#Host Enrichment - -bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.host.enrichment_tag=host - - -#GeoEnrichment - -bolt.enrichment.geo.enrichment_tag=geo -bolt.enrichment.geo.adapter.table=GEO -bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr - -#WhoisEnrichment - -bolt.enrichment.whois.hbase.table.name=whois -bolt.enrichment.whois.enrichment_tag=whois -bolt.enrichment.whois.fields=host,query -bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10 - -#CIF Enrichment -bolt.enrichment.cif.tablename=cif_table -bolt.enrichment.cif.fields.host=host,query -bolt.enrichment.cif.fields.email=email -bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr -bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.cif.enrichment_tag=cif - -#Threat Enrichment -bolt.enrichment.threat.tablename=threat_table -bolt.enrichment.threat.fields=host,query,ip_src_addr,ip_dst_addr -bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.threat.enrichment_tag=threat - -#Indexing Bolt -bolt.indexing.indexname=bro_index -bolt.indexing.timestamp=yyyy.MM.ww -bolt.indexing.documentname=bro_doc -bolt.indexing.bulk=200 -bolt.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Indexing Bolt -bolt.alerts.indexing.indexname=alert -bolt.alerts.indexing.documentname=bro_alert -bolt.alerts.indexing.timestamp=yyyy.MM.ww -bolt.alerts.indexing.bulk=1 -bolt.alerts.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Error Indexing Bolt -bolt.error.indexing.indexname=error -bolt.error.indexing.timestamp=yyyy.MM -bolt.error.indexing.documentname=bro_error -bolt.error.indexing.bulk=1 -bolt.error.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Bolt - - -bolt.alerts.adapter=com.opensoc.alerts.adapters.ThreatAlertsAdapter -com.opensoc.alerts.adapters.ThreatAlertsAdapter.enrichment_tag=Threat_Enrichment -com.opensoc.alerts.adapters.ThreatAlertsAdapter.whitelist_table_name = ip_whitelist -com.opensoc.alerts.adapters.ThreatAlertsAdapter.blacklist_table_name = ip_blacklist -com.opensoc.alerts.adapters.ThreatAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3 -com.opensoc.alerts.adapters.ThreatAlertsAdapter.port=2181 -com.opensoc.alerts.adapters.ThreatAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600 -com.opensoc.alerts.adapters.ThreatAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000 - - -#bolt.alerts.adapter=com.opensoc.alerts.adapters.CIFAlertsAdapter -#com.opensoc.alerts.adapters.CIFAlertsAdapter.enrichment_tag=CIF_Enrichment -#com.opensoc.alerts.adapters.CIFAlertsAdapter.whitelist_table_name = ip_whitelist -#com.opensoc.alerts.adapters.CIFAlertsAdapter.blacklist_table_name = ip_blacklist -#com.opensoc.alerts.adapters.CIFAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3 -#com.opensoc.alerts.adapters.CIFAlertsAdapter.port=2181 -#com.opensoc.alerts.adapters.CIFAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600 -#com.opensoc.alerts.adapters.CIFAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000 - -#bolt.alerts.adapter=com.opensoc.alerts.adapters.HbaseWhiteAndBlacklistAdapter -#com.opensoc.alerts.adapters.HbaseWhiteAndBlacklistAdapter.whitelist_table_name = ip_whitelist -#com.opensoc.alerts.adapters.HbaseWhiteAndBlacklistAdapter.blacklist_table_name = ip_blacklist -#com.opensoc.alerts.adapters.HbaseWhiteAndBlacklistAdapter.quorum=zkpr1,zkpr2,zkpr3 -#com.opensoc.alerts.adapters.HbaseWhiteAndBlacklistAdapter.port=2181 -#com.opensoc.alerts.adapters.HbaseWhiteAndBlacklistAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600 -#com.opensoc.alerts.adapters.HbaseWhiteAndBlacklistAdapter._MAX_TIME_RETAIN_MINUTES=1000 - - - - - -#HDFS Bolt -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://nn1:8020 -bolt.hdfs.wip.file.path=/bro/wip -bolt.hdfs.finished.file.path=/bro/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec - -#Kafka Bolt -bolt.kafka.topic=bro_enriched \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/topology_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/topology_identifier.conf deleted file mode 100644 index bb72783d8c..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/bro/topology_identifier.conf +++ /dev/null @@ -1,4 +0,0 @@ -#Each topology must have a unique identifier. This setting is required - -topology.id=bro -instance.id=B001 \ No newline at end of file From 0237f17958bef476735f3e87fe862a7ed219ad7b Mon Sep 17 00:00:00 2001 From: Sean Schulte Date: Tue, 12 Jan 2016 15:38:09 -0600 Subject: [PATCH 4/7] Added Flux yaml for existing topologies. --- .../OpenSOC_Configs/topologies/asa/local.yaml | 358 +++++++++++++++++ .../topologies/asa/remote.yaml | 372 ++++++++++++++++++ .../topologies/fireeye/local.yaml | 358 +++++++++++++++++ .../topologies/fireeye/remote.yaml | 372 ++++++++++++++++++ .../OpenSOC_Configs/topologies/ise/local.yaml | 185 +++++++++ .../topologies/ise/remote.yaml | 199 ++++++++++ .../topologies/lancope/local.yaml | 320 +++++++++++++++ .../topologies/lancope/remote.yaml | 334 ++++++++++++++++ .../topologies/paloalto/local.yaml | 165 ++++++++ .../topologies/paloalto/remote.yaml | 179 +++++++++ .../topologies/pcap/local.yaml | 171 ++++++++ .../topologies/pcap/remote.yaml | 185 +++++++++ .../topologies/sourcefire/local.yaml | 358 +++++++++++++++++ .../topologies/sourcefire/remote.yaml | 372 ++++++++++++++++++ 14 files changed, 3928 insertions(+) create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/local.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/remote.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/local.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/remote.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/local.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/remote.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/local.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/remote.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/local.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/remote.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/local.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/remote.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/local.yaml create mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/remote.yaml diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/local.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/local.yaml new file mode 100644 index 0000000000..253f28dbb4 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/local.yaml @@ -0,0 +1,358 @@ +name: "asa-local" +config: + topology.workers: 1 + +components: + - id: "asaParser" + className: "com.opensoc.parsing.parsers.GrokAsaParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "geoKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "geoEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter" + constructorArgs: + - "${mysql.ip}" + - ${mysql.port} + - "${mysql.username}" + - "${mysql.password}" + - "GEO" + - id: "hostsKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "hostEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.host.HostFromJSONListAdapter" + constructorArgs: + - '${com.opensoc.enrichment.host.known_hosts}' + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsConfig" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: ["whitelist_table_name", "ip_whitelist"] + - name: "put" + args: ["blacklist_table_name", "ip_blacklist"] + - name: "put" + args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"] + - name: "put" + args: ["port", "2181"] + - name: "put" + args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"] + - name: "put" + args: ["_MAX_TIME_RETAIN_MINUTES", "1000"] + - id: "alertsAdapter" + className: "com.opensoc.alerts.adapters.CIFAlertsAdapter" + constructorArgs: + - ref: "alertsConfig" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "asa"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "com.opensoc.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/AsaOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "asaParser" + - name: "withOutputFieldName" + args: + - "asa" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "geoEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["geo"] + - name: "withAdapter" + args: + - ref: "geoEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["asa"] + - name: "withKeys" + args: + - ref: "geoKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "hostEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["host"] + - name: "withAdapter" + args: + - ref: "hostEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["asa"] + - name: "withKeys" + args: + - ref: "hostsKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "asa_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "asa_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "com.opensoc.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "asa_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "asa_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> geo" + from: "parserBolt" + to: "geoEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "geo -> host" + from: "geoEnrichmentBolt" + to: "hostEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "host -> alerts" + from: "hostEnrichmentBolt" + to: "alertsBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "alerts -> indexing" + from: "alertsBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/remote.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/remote.yaml new file mode 100644 index 0000000000..b7013b5ede --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/remote.yaml @@ -0,0 +1,372 @@ +name: "asa" +config: + topology.workers: 1 + +components: + - id: "asaParser" + className: "com.opensoc.parsing.parsers.GrokAsaParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "geoKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "geoEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter" + constructorArgs: + - "${mysql.ip}" + - ${mysql.port} + - "${mysql.username}" + - "${mysql.password}" + - "GEO" + - id: "hostsKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "hostEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.host.HostFromJSONListAdapter" + constructorArgs: + - '${com.opensoc.enrichment.host.known_hosts}' + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsConfig" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: ["whitelist_table_name", "ip_whitelist"] + - name: "put" + args: ["blacklist_table_name", "ip_blacklist"] + - name: "put" + args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"] + - name: "put" + args: ["port", "2181"] + - name: "put" + args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"] + - name: "put" + args: ["_MAX_TIME_RETAIN_MINUTES", "1000"] + - id: "alertsAdapter" + className: "com.opensoc.alerts.adapters.CIFAlertsAdapter" + constructorArgs: + - ref: "alertsConfig" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "asa"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "forceFromStart" + value: true + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "asaParser" + - name: "withOutputFieldName" + args: + - "asa" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "geoEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["geo"] + - name: "withAdapter" + args: + - ref: "geoEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["asa"] + - name: "withKeys" + args: + - ref: "geoKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "hostEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["host"] + - name: "withAdapter" + args: + - ref: "hostEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["asa"] + - name: "withKeys" + args: + - ref: "hostsKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "asa_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "asa_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "com.opensoc.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "asa_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "asa_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> geo" + from: "parserBolt" + to: "geoEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "geo -> host" + from: "geoEnrichmentBolt" + to: "hostEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "host -> alerts" + from: "hostEnrichmentBolt" + to: "alertsBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "alerts -> indexing" + from: "alertsBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/local.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/local.yaml new file mode 100644 index 0000000000..841e69c799 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/local.yaml @@ -0,0 +1,358 @@ +name: "fireeye-local" +config: + topology.workers: 1 + +components: + - id: "fireEyeParser" + className: "com.opensoc.parsing.parsers.BasicFireEyeParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "geoKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "geoEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter" + constructorArgs: + - "${mysql.ip}" + - ${mysql.port} + - "${mysql.username}" + - "${mysql.password}" + - "GEO" + - id: "hostsKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "hostEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.host.HostFromJSONListAdapter" + constructorArgs: + - '${com.opensoc.enrichment.host.known_hosts}' + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsConfig" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: ["whitelist_table_name", "ip_whitelist"] + - name: "put" + args: ["blacklist_table_name", "ip_blacklist"] + - name: "put" + args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"] + - name: "put" + args: ["port", "2181"] + - name: "put" + args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"] + - name: "put" + args: ["_MAX_TIME_RETAIN_MINUTES", "1000"] + - id: "alertsAdapter" + className: "com.opensoc.alerts.adapters.CIFAlertsAdapter" + constructorArgs: + - ref: "alertsConfig" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "fireeye"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "com.opensoc.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/FireeyeExampleOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "fireEyeParser" + - name: "withOutputFieldName" + args: + - "fireeye" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "geoEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["geo"] + - name: "withAdapter" + args: + - ref: "geoEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["fireeye"] + - name: "withKeys" + args: + - ref: "geoKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "hostEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["host"] + - name: "withAdapter" + args: + - ref: "hostEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["fireeye"] + - name: "withKeys" + args: + - ref: "hostsKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "fireeye_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "fireeye_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "com.opensoc.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "fireeye_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "fireeye_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> geo" + from: "parserBolt" + to: "geoEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "geo -> host" + from: "geoEnrichmentBolt" + to: "hostEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "host -> alerts" + from: "hostEnrichmentBolt" + to: "alertsBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "alerts -> indexing" + from: "alertsBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/remote.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/remote.yaml new file mode 100644 index 0000000000..0f7bf6ad52 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/remote.yaml @@ -0,0 +1,372 @@ +name: "fireeye" +config: + topology.workers: 1 + +components: + - id: "fireEyeParser" + className: "com.opensoc.parsing.parsers.BasicFireEyeParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "geoKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "geoEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter" + constructorArgs: + - "${mysql.ip}" + - ${mysql.port} + - "${mysql.username}" + - "${mysql.password}" + - "GEO" + - id: "hostsKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "hostEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.host.HostFromJSONListAdapter" + constructorArgs: + - '${com.opensoc.enrichment.host.known_hosts}' + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsConfig" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: ["whitelist_table_name", "ip_whitelist"] + - name: "put" + args: ["blacklist_table_name", "ip_blacklist"] + - name: "put" + args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"] + - name: "put" + args: ["port", "2181"] + - name: "put" + args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"] + - name: "put" + args: ["_MAX_TIME_RETAIN_MINUTES", "1000"] + - id: "alertsAdapter" + className: "com.opensoc.alerts.adapters.CIFAlertsAdapter" + constructorArgs: + - ref: "alertsConfig" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "fireeye"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "forceFromStart" + value: true + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "fireEyeParser" + - name: "withOutputFieldName" + args: + - "fireeye" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "geoEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["geo"] + - name: "withAdapter" + args: + - ref: "geoEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["fireeye"] + - name: "withKeys" + args: + - ref: "geoKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "hostEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["host"] + - name: "withAdapter" + args: + - ref: "hostEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["fireeye"] + - name: "withKeys" + args: + - ref: "hostsKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "fireeye_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "fireeye_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "com.opensoc.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "fireeye_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "fireeye_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> geo" + from: "parserBolt" + to: "geoEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "geo -> host" + from: "geoEnrichmentBolt" + to: "hostEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "host -> alerts" + from: "hostEnrichmentBolt" + to: "alertsBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "alerts -> indexing" + from: "alertsBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/local.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/local.yaml new file mode 100644 index 0000000000..041b614284 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/local.yaml @@ -0,0 +1,185 @@ +name: "ise-local" +config: + topology.workers: 1 + +components: + - id: "iseParser" + className: "com.opensoc.parsing.parsers.BasicIseParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "com.opensoc.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/ISESampleOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "iseParser" + - name: "withOutputFieldName" + args: + - "ise" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "ise_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "ise_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "ise_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> indexing" + from: "parserBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/remote.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/remote.yaml new file mode 100644 index 0000000000..c44b4c6efe --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/remote.yaml @@ -0,0 +1,199 @@ +name: "ise" +config: + topology.workers: 1 + +components: + - id: "iseParser" + className: "com.opensoc.parsing.parsers.BasicIseParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "forceFromStart" + value: true + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "iseParser" + - name: "withOutputFieldName" + args: + - "ise" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "ise_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "ise_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "ise_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> indexing" + from: "parserBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/local.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/local.yaml new file mode 100644 index 0000000000..b8c353d4c1 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/local.yaml @@ -0,0 +1,320 @@ +name: "lancope-local" +config: + topology.workers: 1 + +components: + - id: "lancopeParser" + className: "com.opensoc.parsing.parsers.BasicLancopeParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "geoKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "geoEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter" + constructorArgs: + - "${mysql.ip}" + - ${mysql.port} + - "${mysql.username}" + - "${mysql.password}" + - "GEO" + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsConfig" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: ["whitelist_table_name", "ip_whitelist"] + - name: "put" + args: ["blacklist_table_name", "ip_blacklist"] + - name: "put" + args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"] + - name: "put" + args: ["port", "2181"] + - name: "put" + args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"] + - name: "put" + args: ["_MAX_TIME_RETAIN_MINUTES", "1000"] + - id: "alertsAdapter" + className: "com.opensoc.alerts.adapters.CIFAlertsAdapter" + constructorArgs: + - ref: "alertsConfig" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "lancope"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "com.opensoc.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/LancopeExampleOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "lancopeParser" + - name: "withOutputFieldName" + args: + - "lancope" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "geoEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["geo"] + - name: "withAdapter" + args: + - ref: "geoEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["lancope"] + - name: "withKeys" + args: + - ref: "geoKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "lancope_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "lancope_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "com.opensoc.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "lancope_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "lancope_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> geo" + from: "parserBolt" + to: "geoEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "geo -> alerts" + from: "geoEnrichmentBolt" + to: "alertsBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "alerts -> indexing" + from: "alertsBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/remote.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/remote.yaml new file mode 100644 index 0000000000..c7cc60eca6 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/remote.yaml @@ -0,0 +1,334 @@ +name: "lancope" +config: + topology.workers: 1 + +components: + - id: "lancopeParser" + className: "com.opensoc.parsing.parsers.BasicLancopeParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "geoKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "geoEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter" + constructorArgs: + - "${mysql.ip}" + - ${mysql.port} + - "${mysql.username}" + - "${mysql.password}" + - "GEO" + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsConfig" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: ["whitelist_table_name", "ip_whitelist"] + - name: "put" + args: ["blacklist_table_name", "ip_blacklist"] + - name: "put" + args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"] + - name: "put" + args: ["port", "2181"] + - name: "put" + args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"] + - name: "put" + args: ["_MAX_TIME_RETAIN_MINUTES", "1000"] + - id: "alertsAdapter" + className: "com.opensoc.alerts.adapters.CIFAlertsAdapter" + constructorArgs: + - ref: "alertsConfig" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "lancope"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "forceFromStart" + value: true + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "lancopeParser" + - name: "withOutputFieldName" + args: + - "lancope" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "geoEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["geo"] + - name: "withAdapter" + args: + - ref: "geoEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["lancope"] + - name: "withKeys" + args: + - ref: "geoKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "lancope_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "lancope_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "com.opensoc.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "lancope_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "lancope_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> geo" + from: "parserBolt" + to: "geoEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "geo -> alerts" + from: "geoEnrichmentBolt" + to: "alertsBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "alerts -> indexing" + from: "alertsBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/local.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/local.yaml new file mode 100644 index 0000000000..1f05dd3f0f --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/local.yaml @@ -0,0 +1,165 @@ +name: "paloalto-local" +config: + topology.workers: 1 + +components: + - id: "paloAltoParser" + className: "com.opensoc.parsing.parsers.BasicPaloAltoFirewallParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "filenameFormat" + className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat" + configMethods: + - name: "withPath" + args: + - "${bolt.hdfs.wip.file.path}" + - id: "messageField" + className: "backtype.storm.tuple.Fields" + constructorArgs: + - ["message"] + - id: "recordFormat" + className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat" + configMethods: + - name: "withFieldDelimiter" + args: + - "${bolt.hdfs.field.delimiter}" + - name: "withFields" + args: + - ref: "messageField" + - id: "rotationPolicy" + className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy" + constructorArgs: + - ${bolt.hdfs.file.rotation.size.in.mb} + - MB + - id: "syncPolicy" + className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy" + constructorArgs: + - ${bolt.hdfs.batch.size} + - id: "moveFileAction" + className: "org.apache.storm.hdfs.common.rotation.MoveFileAction" + configMethods: + - name: "toDestination" + args: + - "${bolt.hdfs.finished.file.path}" + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "com.opensoc.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/PaloaltoOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "paloAltoParser" + - name: "withOutputFieldName" + args: + - "paloalto" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "hdfsBolt" + className: "org.apache.storm.hdfs.bolt.HdfsBolt" + configMethods: + - name: "withFsUrl" + args: + - "${bolt.hdfs.file.system.url}" + - name: "withFileNameFormat" + args: + - ref: "filenameFormat" + - name: "withRecordFormat" + args: + - ref: "recordFormat" + - name: "withRotationPolicy" + args: + - ref: "rotationPolicy" + - name: "withSyncPolicy" + args: + - ref: "syncPolicy" + - name: "addRotationAction" + args: + - ref: "moveFileAction" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> hdfs" + from: "parserBolt" + to: "hdfsBolt" + grouping: + streamId: "message" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/remote.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/remote.yaml new file mode 100644 index 0000000000..a9e68c82a8 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/remote.yaml @@ -0,0 +1,179 @@ +name: "paloalto" +config: + topology.workers: 1 + +components: + - id: "paloAltoParser" + className: "com.opensoc.parsing.parsers.BasicPaloAltoFirewallParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "filenameFormat" + className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat" + configMethods: + - name: "withPath" + args: + - "${bolt.hdfs.wip.file.path}" + - id: "messageField" + className: "backtype.storm.tuple.Fields" + constructorArgs: + - ["message"] + - id: "recordFormat" + className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat" + configMethods: + - name: "withFieldDelimiter" + args: + - "${bolt.hdfs.field.delimiter}" + - name: "withFields" + args: + - ref: "messageField" + - id: "rotationPolicy" + className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy" + constructorArgs: + - ${bolt.hdfs.file.rotation.size.in.mb} + - MB + - id: "syncPolicy" + className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy" + constructorArgs: + - ${bolt.hdfs.batch.size} + - id: "moveFileAction" + className: "org.apache.storm.hdfs.common.rotation.MoveFileAction" + configMethods: + - name: "toDestination" + args: + - "${bolt.hdfs.finished.file.path}" + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "forceFromStart" + value: true + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "paloAltoParser" + - name: "withOutputFieldName" + args: + - "paloalto" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "hdfsBolt" + className: "org.apache.storm.hdfs.bolt.HdfsBolt" + configMethods: + - name: "withFsUrl" + args: + - "${bolt.hdfs.file.system.url}" + - name: "withFileNameFormat" + args: + - ref: "filenameFormat" + - name: "withRecordFormat" + args: + - ref: "recordFormat" + - name: "withRotationPolicy" + args: + - ref: "rotationPolicy" + - name: "withSyncPolicy" + args: + - ref: "syncPolicy" + - name: "addRotationAction" + args: + - ref: "moveFileAction" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> hdfs" + from: "parserBolt" + to: "hdfsBolt" + grouping: + streamId: "message" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/local.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/local.yaml new file mode 100644 index 0000000000..9102c183ac --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/local.yaml @@ -0,0 +1,171 @@ +name: "pcap-local" +config: + topology.workers: 1 + +components: + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "com.opensoc.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/PCAPExampleOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.PcapParserBolt" + configMethods: + - name: "withTsPrecision" + args: ["MICRO"] + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "pcap_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "pcap_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "pcap_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> indexing" + from: "parserBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/remote.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/remote.yaml new file mode 100644 index 0000000000..0e03bdc676 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/remote.yaml @@ -0,0 +1,185 @@ +name: "pcap" +config: + topology.workers: 1 + +components: + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "forceFromStart" + value: true + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.PcapParserBolt" + configMethods: + - name: "withTsPrecision" + args: ["MICRO"] + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "pcap_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "pcap_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "pcap_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> indexing" + from: "parserBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/local.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/local.yaml new file mode 100644 index 0000000000..28fca59aa8 --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/local.yaml @@ -0,0 +1,358 @@ +name: "sourcefire-local" +config: + topology.workers: 1 + +components: + - id: "sourceFireParser" + className: "com.opensoc.parsing.parsers.BasicSourcefireParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "geoKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "geoEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter" + constructorArgs: + - "${mysql.ip}" + - ${mysql.port} + - "${mysql.username}" + - "${mysql.password}" + - "GEO" + - id: "hostsKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "hostEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.host.HostFromJSONListAdapter" + constructorArgs: + - '${com.opensoc.enrichment.host.known_hosts}' + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsConfig" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: ["whitelist_table_name", "ip_whitelist"] + - name: "put" + args: ["blacklist_table_name", "ip_blacklist"] + - name: "put" + args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"] + - name: "put" + args: ["port", "2181"] + - name: "put" + args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"] + - name: "put" + args: ["_MAX_TIME_RETAIN_MINUTES", "1000"] + - id: "alertsAdapter" + className: "com.opensoc.alerts.adapters.CIFAlertsAdapter" + constructorArgs: + - ref: "alertsConfig" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "sourcefire"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "com.opensoc.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/SourcefireExampleOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "sourceFireParser" + - name: "withOutputFieldName" + args: + - "sourcefire" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "geoEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["geo"] + - name: "withAdapter" + args: + - ref: "geoEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["sourcefire"] + - name: "withKeys" + args: + - ref: "geoKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "hostEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["host"] + - name: "withAdapter" + args: + - ref: "hostEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["sourcefire"] + - name: "withKeys" + args: + - ref: "hostsKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "sourcefire_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "sourcefire_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "com.opensoc.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "sourcefire_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "sourcefire_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> geo" + from: "parserBolt" + to: "geoEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "geo -> host" + from: "geoEnrichmentBolt" + to: "hostEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "host -> alerts" + from: "hostEnrichmentBolt" + to: "alertsBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "alerts -> indexing" + from: "alertsBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/remote.yaml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/remote.yaml new file mode 100644 index 0000000000..c5ad6402cd --- /dev/null +++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/remote.yaml @@ -0,0 +1,372 @@ +name: "sourcefire" +config: + topology.workers: 1 + +components: + - id: "sourceFireParser" + className: "com.opensoc.parsing.parsers.BasicSourcefireParser" + - id: "genericMessageFilter" + className: "com.opensoc.filters.GenericMessageFilter" + - id: "geoKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "geoEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter" + constructorArgs: + - "${mysql.ip}" + - ${mysql.port} + - "${mysql.username}" + - "${mysql.password}" + - "GEO" + - id: "hostsKeys" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: ["ip_src_addr"] + - name: "add" + args: ["ip_dst_addr"] + - id: "hostEnrichmentAdapter" + className: "com.opensoc.enrichment.adapters.host.HostFromJSONListAdapter" + constructorArgs: + - '${com.opensoc.enrichment.host.known_hosts}' + - id: "indexAdapter" + className: "com.opensoc.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsConfig" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: ["whitelist_table_name", "ip_whitelist"] + - name: "put" + args: ["blacklist_table_name", "ip_blacklist"] + - name: "put" + args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"] + - name: "put" + args: ["port", "2181"] + - name: "put" + args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"] + - name: "put" + args: ["_MAX_TIME_RETAIN_MINUTES", "1000"] + - id: "alertsAdapter" + className: "com.opensoc.alerts.adapters.CIFAlertsAdapter" + constructorArgs: + - ref: "alertsConfig" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "sourcefire"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.graphite" + - "${com.opensoc.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.console" + - "${com.opensoc.metrics.reporter.console}" + - name: "setProperty" + args: + - "com.opensoc.metrics.reporter.jmx" + - "${com.opensoc.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.address" + - "${com.opensoc.metrics.graphite.address}" + - name: "setProperty" + args: + - "com.opensoc.metrics.graphite.port" + - "${com.opensoc.metrics.graphite.port}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.acks" + - "${com.opensoc.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.emits" + - "${com.opensoc.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryParserBolt.fails" + - "${com.opensoc.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.acks" + - "${com.opensoc.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.emits" + - "${com.opensoc.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.GenericEnrichmentBolt.fails" + - "${com.opensoc.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.acks" + - "${com.opensoc.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.emits" + - "${com.opensoc.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "com.opensoc.metrics.TelemetryIndexingBolt.fails" + - "${com.opensoc.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "forceFromStart" + value: true + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "com.opensoc.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "sourceFireParser" + - name: "withOutputFieldName" + args: + - "sourcefire" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "geoEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["geo"] + - name: "withAdapter" + args: + - ref: "geoEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["sourcefire"] + - name: "withKeys" + args: + - ref: "geoKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "hostEnrichmentBolt" + className: "com.opensoc.enrichment.common.GenericEnrichmentBolt" + configMethods: + - name: "withEnrichmentTag" + args: ["host"] + - name: "withAdapter" + args: + - ref: "hostEnrichmentAdapter" + - name: "withMaxTimeRetain" + args: [10] + - name: "withMaxCacheSize" + args: [10000] + - name: "withOutputFieldName" + args: ["sourcefire"] + - name: "withKeys" + args: + - ref: "hostsKeys" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "sourcefire_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "sourcefire_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "com.opensoc.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "sourcefire_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "com.opensoc.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "sourcefire_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> geo" + from: "parserBolt" + to: "geoEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "geo -> host" + from: "geoEnrichmentBolt" + to: "hostEnrichmentBolt" + grouping: + type: FIELDS + streamId: "message" + args: ["key"] + - name: "host -> alerts" + from: "hostEnrichmentBolt" + to: "alertsBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "alerts -> indexing" + from: "alertsBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE From b92766d48f718510ca8e6af98db29c2e4f139e92 Mon Sep 17 00:00:00 2001 From: Sean Schulte Date: Wed, 13 Jan 2016 11:04:57 -0600 Subject: [PATCH 5/7] Removed old config files. --- .../etc/env/environment_common.conf | 4 - .../etc/env/es_connection.conf | 3 - .../etc/env/hdfs_connection.conf | 2 - .../etc/env/mysql_connection.conf | 4 - .../topologies/asa/features_enabled.conf | 113 ------------- .../topologies/asa/metrics.conf | 26 --- .../topologies/asa/topology.conf | 110 ------------- .../topologies/asa/topology_identifier.conf | 4 - .../topologies/environment_identifier.conf | 5 - .../topologies/fireeye/features_enabled.conf | 113 ------------- .../topologies/fireeye/metrics.conf | 26 --- .../topologies/fireeye/topology.conf | 110 ------------- .../fireeye/topology_identifier.conf | 4 - .../OpenSOC_Configs/topologies/ise/alerts.xml | 11 -- .../topologies/ise/features_enabled.conf | 113 ------------- .../topologies/ise/metrics.conf | 26 --- .../topologies/ise/topology.conf | 101 ------------ .../topologies/ise/topology_identifier.conf | 4 - .../topologies/lancope/alerts.xml | 11 -- .../topologies/lancope/features_enabled.conf | 113 ------------- .../topologies/lancope/metrics.conf | 26 --- .../topologies/lancope/topology.conf | 101 ------------ .../lancope/topology_identifier.conf | 4 - .../topologies/paloalto/features_enabled.conf | 113 ------------- .../topologies/paloalto/metrics.conf | 26 --- .../topologies/paloalto/topology.conf | 113 ------------- .../paloalto/topology_identifier.conf | 4 - .../topologies/pcap/features_enabled.conf | 118 -------------- .../topologies/pcap/metrics.conf | 26 --- .../topologies/pcap/topology.conf | 150 ------------------ .../topologies/pcap/topology_identifier.conf | 4 - .../topologies/sourcefire/alerts.xml | 11 -- .../sourcefire/features_enabled.conf | 113 ------------- .../topologies/sourcefire/metrics.conf | 26 --- .../topologies/sourcefire/topology.conf | 110 ------------- .../sourcefire/topology_identifier.conf | 4 - 36 files changed, 1852 deletions(-) delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/environment_common.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/es_connection.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/hdfs_connection.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/mysql_connection.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/features_enabled.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/metrics.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/topology.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/topology_identifier.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/environment_identifier.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/features_enabled.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/metrics.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/topology.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/topology_identifier.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/alerts.xml delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/features_enabled.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/metrics.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/topology.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/topology_identifier.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/alerts.xml delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/features_enabled.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/metrics.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/topology.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/topology_identifier.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/features_enabled.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/metrics.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/topology.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/topology_identifier.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/features_enabled.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/metrics.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/topology.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/topology_identifier.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/alerts.xml delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/features_enabled.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/metrics.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/topology.conf delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/topology_identifier.conf diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/environment_common.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/environment_common.conf deleted file mode 100644 index f93921a308..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/environment_common.conf +++ /dev/null @@ -1,4 +0,0 @@ -kafka.zk.port=2181 -kafka.zk.list=zkpr1,zkpr2,zkpr3 -kafka.zk=zkpr1:2181,zkpr2:2181,zkpr3:2181 -kafka.br=dn01:9092,dn02:9092,dn03:9092,dn04:9092,dn05:9092,dn06:9092,dn07:9092,dn08:9092 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/es_connection.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/es_connection.conf deleted file mode 100644 index bb8861c061..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/es_connection.conf +++ /dev/null @@ -1,3 +0,0 @@ -es.ip=esp -es.port=9300 -es.clustername=devo_es \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/hdfs_connection.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/hdfs_connection.conf deleted file mode 100644 index f7e7f2bdbf..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/hdfs_connection.conf +++ /dev/null @@ -1,2 +0,0 @@ -bolt.hdfs.IP=nn1 -bolt.hdfs.port=8020 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/mysql_connection.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/mysql_connection.conf deleted file mode 100644 index 15690af98f..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/etc/env/mysql_connection.conf +++ /dev/null @@ -1,4 +0,0 @@ -mysql.ip=mysql -mysql.port=0 -mysql.username=test -mysql.password=123123 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/features_enabled.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/features_enabled.conf deleted file mode 100644 index 5b45ddef9a..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/features_enabled.conf +++ /dev/null @@ -1,113 +0,0 @@ -#Enable and disable features for each topology - -#Feature: Test spout -##Feature Description: Reads telemetry from file and ingests it into topology. Used for testing or bulk loading the topology - -spout.test.name=TestSpout -spout.test.enabled=true -spout.test.num.tasks=1 -spout.test.parallelism.hint=1 - -#Feature: Kafka spout -##Feature Description: Acts as a Kafka consumer. Takes messages from a Kafka topic and ingests them into a topology - -spout.kafka.name=KafkaSpout -spout.kafka.enabled=false -spout.kafka.num.tasks=1 -spout.kafka.parallelism.hint=1 - -#Feature: Parser Bolt -##Feature Description: Parses telemetry from its native format into a native JSON - -bolt.parser.name=ParserBolt -bolt.parser.enabled=true -bolt.parser.num.tasks=1 -bolt.parser.parallelism.hint=1 - -#Feature: Host Enrichment -##Feature Description: Appends information about known hosts to a telemetry message - -bolt.enrichment.host.name=HostEnrichment -bolt.enrichment.host.enabled=true -bolt.enrichment.host.num.tasks=1 -bolt.enrichment.host.parallelism.hint=1 - -#Feature: Geo Enrichment -##Feature Description: Appends geo information about known non-local IPs to a telemetry message - -bolt.enrichment.geo.name=GeoEnrichment -bolt.enrichment.geo.enabled=true -bolt.enrichment.geo.num.tasks=1 -bolt.enrichment.geo.parallelism.hint=1 - -#Feature: Whois Enrichment -##Feature Description: Appends whois information about known domains to a telemetry message - -bolt.enrichment.whois.name=WhoisEnrichment -bolt.enrichment.whois.enabled=false -bolt.enrichment.whois.num.tasks=1 -bolt.enrichment.whois.parallelism.hint=1 - -#Feature: CIF Enrichment -##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message - -bolt.enrichment.cif.name=SIFBolt -bolt.enrichment.cif.enabled=false -bolt.enrichment.cif.num.tasks=1 -bolt.enrichment.cif.parallelism.hint=1 - -#Feature: Threat Enrichment -##Feature Description: Appends information from Threat intelligence feeds to a telemetry message - -bolt.enrichment.threat.name=ThreatBolt -bolt.enrichment.threat.enabled=false -bolt.enrichment.threat.num.tasks=1 -bolt.enrichment.threat.parallelism.hint=1 - -#Feature: Rules-Based Alerts -##Feature Description: Tags messages with rules-based alerts - -bolt.alerts.name=Alerts -bolt.alerts.enabled=true -bolt.alerts.num.tasks=1 -bolt.alerts.parallelism.hint=1 - -#Feature: Indexer -##Feature Description: Indexes telemetry messages in ElasticSearch or Solr - -bolt.indexing.name=IndexBolt -bolt.indexing.enabled=true -bolt.indexing.num.tasks=1 -bolt.indexing.parallelism.hint=1 - -#Feature: Alerts Indexer -##Feature Description: Indexes alert messages in ElasticSearch or Solr - -bolt.alerts.indexing.name=AlertIndexBolt -bolt.alerts.indexing.enabled=true -bolt.alerts.indexing.num.tasks=1 -bolt.alerts.indexing.parallelism.hint=1 - -#Feature: Error Indexer -##Feature Description: Indexes error messages in ElasticSearch or Solr - -bolt.error.indexing.name=ErrorIndexBolt -bolt.error.indexing.enabled=true -bolt.error.indexing.num.tasks=1 -bolt.error.indexing.parallelism.hint=1 - -#Feature: Kafka Bolt -##Feature Description: Writes telemetry messages back into a Kafka topic - -bolt.kafka.name=KafkaBolt -bolt.kafka.enabled=false -bolt.kafka.num.tasks=1 -bolt.kafka.parallelism.hint=1 - -#Feature: HDFS Bolt -##Feature Description: Writes telemetry messages into HDFS - -bolt.hdfs.name=HDFSBolt -bolt.hdfs.enabled=false -bolt.hdfs.num.tasks=1 -bolt.hdfs.parallelism.hint=1 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/metrics.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/metrics.conf deleted file mode 100644 index 1daef3d889..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/metrics.conf +++ /dev/null @@ -1,26 +0,0 @@ -#reporters -com.opensoc.metrics.reporter.graphite=true -com.opensoc.metrics.reporter.console=false -com.opensoc.metrics.reporter.jmx=false - -#Graphite Addresses - -com.opensoc.metrics.graphite.address=localhost -com.opensoc.metrics.graphite.port=2023 - -#TelemetryParserBolt -com.opensoc.metrics.TelemetryParserBolt.acks=true -com.opensoc.metrics.TelemetryParserBolt.emits=true -com.opensoc.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -com.opensoc.metrics.GenericEnrichmentBolt.acks=true -com.opensoc.metrics.GenericEnrichmentBolt.emits=true -com.opensoc.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -com.opensoc.metrics.TelemetryIndexingBolt.acks=true -com.opensoc.metrics.TelemetryIndexingBolt.emits=true -com.opensoc.metrics.TelemetryIndexingBolt.fails=true diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/topology.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/topology.conf deleted file mode 100644 index 1720632cbf..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/topology.conf +++ /dev/null @@ -1,110 +0,0 @@ -include = ../../etc/env/environment_common.conf -include = ../../etc/env/es_connection.conf -include = ../../etc/env/hdfs_connection.conf -include = ../../etc/env/mysql_connection.conf -include = metrics.conf -include = features_enabled.conf - -#Global Properties - -debug.mode=true -local.mode=true -num.workers=1 - -#Standard 5-tuple fields - -source.ip=ip_src_addr -source.port=ip_src_port -dest.ip=ip_dst_addr -dest.port=ip_dst_port -protocol=protocol - -#Test Spout -spout.test.parallelism.repeat=false - -#Kafka Spout -spout.kafka.topic=asa_raw - -#Parser Bolt -bolt.parser.adapter=com.opensoc.parsing.parsers.GrokAsaParser - -#Host Enrichment - -bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.host.enrichment_tag=host - - -#GeoEnrichment - -bolt.enrichment.geo.enrichment_tag=geo -bolt.enrichment.geo.adapter.table=GEO -bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr - -#WhoisEnrichment - -bolt.enrichment.whois.hbase.table.name=whois -bolt.enrichment.whois.enrichment_tag=whois -bolt.enrichment.whois.fields=host -bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10 - -#CIF Enrichment -bolt.enrichment.cif.tablename=cif_table -bolt.enrichment.cif.fields.host=host -bolt.enrichment.cif.fields.email=email -bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr -bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.cif.enrichment_tag=cif - -#Threat Enrichment -bolt.enrichment.threat.tablename=threat_table -bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr -bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.threat.enrichment_tag=threat - -#Indexing Bolt -bolt.indexing.indexname=asa_index -bolt.indexing.timestamp=yyyy.MM.ww -bolt.indexing.documentname=asa_doc -bolt.indexing.bulk=1 -bolt.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Indexing Bolt -bolt.alerts.indexing.indexname=alert -bolt.alerts.indexing.timestamp.yyyy.MM.ww -bolt.alerts.indexing.documentname=asa_alert -bolt.alerts.indexing.bulk=1 -bolt.alerts.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Error Indexing Bolt -bolt.error.indexing.indexname=error -bolt.error.indexing.timestamp=yyyy.MM -bolt.error.indexing.documentname=asa_error -bolt.error.indexing.bulk=1 -bolt.error.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Bolt -bolt.alerts.adapter=com.opensoc.alerts.adapters.CIFAlertsAdapter -com.opensoc.alerts.adapters.CIFAlertsAdapter.whitelist_table_name = ip_whitelist -com.opensoc.alerts.adapters.CIFAlertsAdapter.blacklist_table_name = ip_blacklist -com.opensoc.alerts.adapters.CIFAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3 -com.opensoc.alerts.adapters.CIFAlertsAdapter.port=2181 -com.opensoc.alerts.adapters.CIFAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600 -com.opensoc.alerts.adapters.CIFAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000 - -#HDFS Bolt -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://nn1:8020 -bolt.hdfs.wip.file.path=/asa/wip -bolt.hdfs.finished.file.path=/asa/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec - -#Kafka Bolt -bolt.kafka.topic=asa_enriched \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/topology_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/topology_identifier.conf deleted file mode 100644 index 68d3463c8b..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/asa/topology_identifier.conf +++ /dev/null @@ -1,4 +0,0 @@ -#Each topology must have a unique identifier. This setting is required - -topology.id=asa -instance.id=A001 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/environment_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/environment_identifier.conf deleted file mode 100644 index 4e8e0059ed..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/environment_identifier.conf +++ /dev/null @@ -1,5 +0,0 @@ -#This file identifies the cluster instance - -customer.id=mtd -datacenter.id=allen -instance.id=dev \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/features_enabled.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/features_enabled.conf deleted file mode 100644 index 5b45ddef9a..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/features_enabled.conf +++ /dev/null @@ -1,113 +0,0 @@ -#Enable and disable features for each topology - -#Feature: Test spout -##Feature Description: Reads telemetry from file and ingests it into topology. Used for testing or bulk loading the topology - -spout.test.name=TestSpout -spout.test.enabled=true -spout.test.num.tasks=1 -spout.test.parallelism.hint=1 - -#Feature: Kafka spout -##Feature Description: Acts as a Kafka consumer. Takes messages from a Kafka topic and ingests them into a topology - -spout.kafka.name=KafkaSpout -spout.kafka.enabled=false -spout.kafka.num.tasks=1 -spout.kafka.parallelism.hint=1 - -#Feature: Parser Bolt -##Feature Description: Parses telemetry from its native format into a native JSON - -bolt.parser.name=ParserBolt -bolt.parser.enabled=true -bolt.parser.num.tasks=1 -bolt.parser.parallelism.hint=1 - -#Feature: Host Enrichment -##Feature Description: Appends information about known hosts to a telemetry message - -bolt.enrichment.host.name=HostEnrichment -bolt.enrichment.host.enabled=true -bolt.enrichment.host.num.tasks=1 -bolt.enrichment.host.parallelism.hint=1 - -#Feature: Geo Enrichment -##Feature Description: Appends geo information about known non-local IPs to a telemetry message - -bolt.enrichment.geo.name=GeoEnrichment -bolt.enrichment.geo.enabled=true -bolt.enrichment.geo.num.tasks=1 -bolt.enrichment.geo.parallelism.hint=1 - -#Feature: Whois Enrichment -##Feature Description: Appends whois information about known domains to a telemetry message - -bolt.enrichment.whois.name=WhoisEnrichment -bolt.enrichment.whois.enabled=false -bolt.enrichment.whois.num.tasks=1 -bolt.enrichment.whois.parallelism.hint=1 - -#Feature: CIF Enrichment -##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message - -bolt.enrichment.cif.name=SIFBolt -bolt.enrichment.cif.enabled=false -bolt.enrichment.cif.num.tasks=1 -bolt.enrichment.cif.parallelism.hint=1 - -#Feature: Threat Enrichment -##Feature Description: Appends information from Threat intelligence feeds to a telemetry message - -bolt.enrichment.threat.name=ThreatBolt -bolt.enrichment.threat.enabled=false -bolt.enrichment.threat.num.tasks=1 -bolt.enrichment.threat.parallelism.hint=1 - -#Feature: Rules-Based Alerts -##Feature Description: Tags messages with rules-based alerts - -bolt.alerts.name=Alerts -bolt.alerts.enabled=true -bolt.alerts.num.tasks=1 -bolt.alerts.parallelism.hint=1 - -#Feature: Indexer -##Feature Description: Indexes telemetry messages in ElasticSearch or Solr - -bolt.indexing.name=IndexBolt -bolt.indexing.enabled=true -bolt.indexing.num.tasks=1 -bolt.indexing.parallelism.hint=1 - -#Feature: Alerts Indexer -##Feature Description: Indexes alert messages in ElasticSearch or Solr - -bolt.alerts.indexing.name=AlertIndexBolt -bolt.alerts.indexing.enabled=true -bolt.alerts.indexing.num.tasks=1 -bolt.alerts.indexing.parallelism.hint=1 - -#Feature: Error Indexer -##Feature Description: Indexes error messages in ElasticSearch or Solr - -bolt.error.indexing.name=ErrorIndexBolt -bolt.error.indexing.enabled=true -bolt.error.indexing.num.tasks=1 -bolt.error.indexing.parallelism.hint=1 - -#Feature: Kafka Bolt -##Feature Description: Writes telemetry messages back into a Kafka topic - -bolt.kafka.name=KafkaBolt -bolt.kafka.enabled=false -bolt.kafka.num.tasks=1 -bolt.kafka.parallelism.hint=1 - -#Feature: HDFS Bolt -##Feature Description: Writes telemetry messages into HDFS - -bolt.hdfs.name=HDFSBolt -bolt.hdfs.enabled=false -bolt.hdfs.num.tasks=1 -bolt.hdfs.parallelism.hint=1 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/metrics.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/metrics.conf deleted file mode 100644 index 1daef3d889..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/metrics.conf +++ /dev/null @@ -1,26 +0,0 @@ -#reporters -com.opensoc.metrics.reporter.graphite=true -com.opensoc.metrics.reporter.console=false -com.opensoc.metrics.reporter.jmx=false - -#Graphite Addresses - -com.opensoc.metrics.graphite.address=localhost -com.opensoc.metrics.graphite.port=2023 - -#TelemetryParserBolt -com.opensoc.metrics.TelemetryParserBolt.acks=true -com.opensoc.metrics.TelemetryParserBolt.emits=true -com.opensoc.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -com.opensoc.metrics.GenericEnrichmentBolt.acks=true -com.opensoc.metrics.GenericEnrichmentBolt.emits=true -com.opensoc.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -com.opensoc.metrics.TelemetryIndexingBolt.acks=true -com.opensoc.metrics.TelemetryIndexingBolt.emits=true -com.opensoc.metrics.TelemetryIndexingBolt.fails=true diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/topology.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/topology.conf deleted file mode 100644 index d50a079114..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/topology.conf +++ /dev/null @@ -1,110 +0,0 @@ -include = ../../etc/env/environment_common.conf -include = ../../etc/env/es_connection.conf -include = ../../etc/env/hdfs_connection.conf -include = ../../etc/env/mysql_connection.conf -include = metrics.conf -include = features_enabled.conf - -#Global Properties - -debug.mode=true -local.mode=true -num.workers=1 - -#Standard 5-tuple fields - -source.ip=ip_src_addr -source.port=ip_src_port -dest.ip=ip_dst_addr -dest.port=ip_dst_port -protocol=protocol - -#Test Spout -spout.test.parallelism.repeat=false - -#Kafka Spout -spout.kafka.topic=fireeye_raw - -#Parser Bolt -bolt.parser.adapter=com.opensoc.parsing.parsers.BasicFireEyeParser - -#Host Enrichment - -bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.host.enrichment_tag=host - - -#GeoEnrichment - -bolt.enrichment.geo.enrichment_tag=geo -bolt.enrichment.geo.adapter.table=GEO -bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr - -#WhoisEnrichment - -bolt.enrichment.whois.hbase.table.name=whois -bolt.enrichment.whois.enrichment_tag=whois -bolt.enrichment.whois.fields=host -bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10 - -#CIF Enrichment -bolt.enrichment.cif.tablename=cif_table -bolt.enrichment.cif.fields.host=host -bolt.enrichment.cif.fields.email=email -bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr -bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.cif.enrichment_tag=cif - -#Threat Enrichment -bolt.enrichment.threat.tablename=threat_table -bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr -bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.threat.enrichment_tag=threat - -#Indexing Bolt -bolt.indexing.indexname=fireeye_index -bolt.indexing.timestamp=yyyy.MM.ww -bolt.indexing.documentname=fireeye_doc -bolt.indexing.bulk=1 -bolt.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Indexing Bolt -bolt.alerts.indexing.indexname=alert -bolt.alerts.indexing.timestamp=yyyy.MM.ww -bolt.alerts.indexing.documentname=fireeye_alert -bolt.alerts.indexing.bulk=1 -bolt.alerts.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Error Indexing Bolt -bolt.error.indexing.indexname=error -bolt.error.indexing.timestamp=yyyy.MM -bolt.error.indexing.documentname=fireeye_error -bolt.error.indexing.bulk=1 -bolt.error.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Bolt -bolt.alerts.adapter=com.opensoc.alerts.adapters.CIFAlertsAdapter -com.opensoc.alerts.adapters.CIFAlertsAdapter.whitelist_table_name = ip_whitelist -com.opensoc.alerts.adapters.CIFAlertsAdapter.blacklist_table_name = ip_blacklist -com.opensoc.alerts.adapters.CIFAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3 -com.opensoc.alerts.adapters.CIFAlertsAdapter.port=2181 -com.opensoc.alerts.adapters.CIFAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600 -com.opensoc.alerts.adapters.CIFAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000 - -#HDFS Bolt -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://nn1:8020 -bolt.hdfs.wip.file.path=/fireeye/wip -bolt.hdfs.finished.file.path=/fireeye/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec - -#Kafka Bolt -bolt.kafka.topic=fireeye_enriched \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/topology_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/topology_identifier.conf deleted file mode 100644 index 3f1e56028f..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/fireeye/topology_identifier.conf +++ /dev/null @@ -1,4 +0,0 @@ -#Each topology must have a unique identifier. This setting is required - -topology.id=fireeye -instance.id=FE001 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/alerts.xml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/alerts.xml deleted file mode 100644 index f36b88178d..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/alerts.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - .*message.* - {"type":"alert","priority":5, "title":"ISE Alert", "body": - "Alert triggered by ISE"} - - - - - diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/features_enabled.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/features_enabled.conf deleted file mode 100644 index 730935d977..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/features_enabled.conf +++ /dev/null @@ -1,113 +0,0 @@ -#Enable and disable features for each topology - -#Feature: Test spout -##Feature Description: Reads telemetry from file and ingests it into topology. Used for testing or bulk loading the topology - -spout.test.name=TestSpout -spout.test.enabled=true -spout.test.num.tasks=1 -spout.test.parallelism.hint=1 - -#Feature: Kafka spout -##Feature Description: Acts as a Kafka consumer. Takes messages from a Kafka topic and ingests them into a topology - -spout.kafka.name=KafkaSpout -spout.kafka.enabled=false -spout.kafka.num.tasks=1 -spout.kafka.parallelism.hint=1 - -#Feature: Parser Bolt -##Feature Description: Parses telemetry from its native format into a native JSON - -bolt.parser.name=ParserBolt -bolt.parser.enabled=true -bolt.parser.num.tasks=1 -bolt.parser.parallelism.hint=1 - -#Feature: Host Enrichment -##Feature Description: Appends information about known hosts to a telemetry message - -bolt.enrichment.host.name=HostEnrichment -bolt.enrichment.host.enabled=false -bolt.enrichment.host.num.tasks=1 -bolt.enrichment.host.parallelism.hint=1 - -#Feature: Geo Enrichment -##Feature Description: Appends geo information about known non-local IPs to a telemetry message - -bolt.enrichment.geo.name=GeoEnrichment -bolt.enrichment.geo.enabled=false -bolt.enrichment.geo.num.tasks=1 -bolt.enrichment.geo.parallelism.hint=1 - -#Feature: Whois Enrichment -##Feature Description: Appends whois information about known domains to a telemetry message - -bolt.enrichment.whois.name=WhoisEnrichment -bolt.enrichment.whois.enabled=false -bolt.enrichment.whois.num.tasks=1 -bolt.enrichment.whois.parallelism.hint=1 - -#Feature: CIF Enrichment -##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message - -bolt.enrichment.cif.name=SIFBolt -bolt.enrichment.cif.enabled=false -bolt.enrichment.cif.num.tasks=1 -bolt.enrichment.cif.parallelism.hint=1 - -#Feature: Threat Enrichment -##Feature Description: Appends information from Threat intelligence feeds to a telemetry message - -bolt.enrichment.threat.name=ThreatBolt -bolt.enrichment.threat.enabled=false -bolt.enrichment.threat.num.tasks=1 -bolt.enrichment.threat.parallelism.hint=1 - -#Feature: Rules-Based Alerts -##Feature Description: Tags messages with rules-based alerts - -bolt.alerts.name=Alerts -bolt.alerts.enabled=false -bolt.alerts.num.tasks=1 -bolt.alerts.parallelism.hint=1 - -#Feature: Indexer -##Feature Description: Indexes telemetry messages in ElasticSearch or Solr - -bolt.indexing.name=IndexBolt -bolt.indexing.enabled=true -bolt.indexing.num.tasks=1 -bolt.indexing.parallelism.hint=1 - -#Feature: Alerts Indexer -##Feature Description: Indexes alert messages in ElasticSearch or Solr - -bolt.alerts.indexing.name=AlertIndexBolt -bolt.alerts.indexing.enabled=true -bolt.alerts.indexing.num.tasks=1 -bolt.alerts.indexing.parallelism.hint=1 - -#Feature: Error Indexer -##Feature Description: Indexes error messages in ElasticSearch or Solr - -bolt.error.indexing.name=ErrorIndexBolt -bolt.error.indexing.enabled=true -bolt.error.indexing.num.tasks=1 -bolt.error.indexing.parallelism.hint=1 - -#Feature: Kafka Bolt -##Feature Description: Writes telemetry messages back into a Kafka topic - -bolt.kafka.name=KafkaBolt -bolt.kafka.enabled=true -bolt.kafka.num.tasks=1 -bolt.kafka.parallelism.hint=1 - -#Feature: HDFS Bolt -##Feature Description: Writes telemetry messages into HDFS - -bolt.hdfs.name=HDFSBolt -bolt.hdfs.enabled=false -bolt.hdfs.num.tasks=1 -bolt.hdfs.parallelism.hint=1 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/metrics.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/metrics.conf deleted file mode 100644 index 1daef3d889..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/metrics.conf +++ /dev/null @@ -1,26 +0,0 @@ -#reporters -com.opensoc.metrics.reporter.graphite=true -com.opensoc.metrics.reporter.console=false -com.opensoc.metrics.reporter.jmx=false - -#Graphite Addresses - -com.opensoc.metrics.graphite.address=localhost -com.opensoc.metrics.graphite.port=2023 - -#TelemetryParserBolt -com.opensoc.metrics.TelemetryParserBolt.acks=true -com.opensoc.metrics.TelemetryParserBolt.emits=true -com.opensoc.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -com.opensoc.metrics.GenericEnrichmentBolt.acks=true -com.opensoc.metrics.GenericEnrichmentBolt.emits=true -com.opensoc.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -com.opensoc.metrics.TelemetryIndexingBolt.acks=true -com.opensoc.metrics.TelemetryIndexingBolt.emits=true -com.opensoc.metrics.TelemetryIndexingBolt.fails=true diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/topology.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/topology.conf deleted file mode 100644 index f986bea67d..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/topology.conf +++ /dev/null @@ -1,101 +0,0 @@ -include = ../../etc/env/environment_common.conf -include = ../../etc/env/es_connection.conf -include = ../../etc/env/hdfs_connection.conf -include = ../../etc/env/mysql_connection.conf -include = metrics.conf -include = features_enabled.conf - -#Global Properties - -debug.mode=true -local.mode=true -num.workers=1 - -#Standard 5-tuple fields - -source.ip=ip_src_addr -source.port=ip_src_port -dest.ip=ip_dst_addr -dest.port=ip_dst_port -protocol=protocol - -#Test Spout -spout.test.parallelism.repeat=false - -#Kafka Spout -spout.kafka.topic=ise_raw - -#Parser Bolt -bolt.parser.adapter=com.opensoc.parsing.parsers.BasicIseParser - -#Host Enrichment - -bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.host.enrichment_tag=host - - -#GeoEnrichment - -bolt.enrichment.geo.enrichment_tag=geo -bolt.enrichment.geo.adapter.table=GEO -bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr - -#WhoisEnrichment - -bolt.enrichment.whois.hbase.table.name=whois -bolt.enrichment.whois.enrichment_tag=whois -bolt.enrichment.whois.fields=host -bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10 - -#CIF Enrichment -bolt.enrichment.cif.tablename=cif_table -bolt.enrichment.cif.fields.host=host -bolt.enrichment.cif.fields.email=email -bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr -bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.cif.enrichment_tag=cif - -#Threat Enrichment -bolt.enrichment.threat.tablename=threat_table -bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr -bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.threat.enrichment_tag=threat - -#Indexing Bolt -bolt.indexing.indexname=ise_index -bolt.indexing.timestamp=yyyy.MM.ww -bolt.indexing.documentname=ise_doc -bolt.indexing.bulk=200 -bolt.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Indexing Bolt -bolt.alerts.indexing.indexname=alert -bolt.alerts.indexing.timestamp=yyyy.MM.ww -bolt.alerts.indexing.documentname=ise_alert -bolt.alerts.indexing.bulk=1 -bolt.alerts.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Error Indexing Bolt -bolt.error.indexing.indexname=error -bolt.error.indexing.timesatmp=yyyy.MM -bolt.error.indexing.documentname=ise_error -bolt.error.indexing.bulk=1 -bolt.error.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#HDFS Bolt -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://nn1:8020 -bolt.hdfs.wip.file.path=/ise/wip -bolt.hdfs.finished.file.path=/ise/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec - -#Kafka Bolt -bolt.kafka.topic=ise_enriched \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/topology_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/topology_identifier.conf deleted file mode 100644 index c500e9f12e..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/ise/topology_identifier.conf +++ /dev/null @@ -1,4 +0,0 @@ -#Each topology must have a unique identifier. This setting is required - -topology.id=ise -instance.id=I001 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/alerts.xml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/alerts.xml deleted file mode 100644 index 368f1c0aff..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/alerts.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - .*message.* - {"type":"alert","priority":5, "title":"Lancope Alert", "body": - "Alert triggered by Lancope"} - - - - - diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/features_enabled.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/features_enabled.conf deleted file mode 100644 index a4dc14d1c6..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/features_enabled.conf +++ /dev/null @@ -1,113 +0,0 @@ -#Enable and disable features for each topology - -#Feature: Test spout -##Feature Description: Reads telemetry from file and ingests it into topology. Used for testing or bulk loading the topology - -spout.test.name=TestSpout -spout.test.enabled=true -spout.test.num.tasks=1 -spout.test.parallelism.hint=1 - -#Feature: Kafka spout -##Feature Description: Acts as a Kafka consumer. Takes messages from a Kafka topic and ingests them into a topology - -spout.kafka.name=KafkaSpout -spout.kafka.enabled=false -spout.kafka.num.tasks=1 -spout.kafka.parallelism.hint=1 - -#Feature: Parser Bolt -##Feature Description: Parses telemetry from its native format into a native JSON - -bolt.parser.name=ParserBolt -bolt.parser.enabled=true -bolt.parser.num.tasks=1 -bolt.parser.parallelism.hint=1 - -#Feature: Host Enrichment -##Feature Description: Appends information about known hosts to a telemetry message - -bolt.enrichment.host.name=HostEnrichment -bolt.enrichment.host.enabled=false -bolt.enrichment.host.num.tasks=1 -bolt.enrichment.host.parallelism.hint=1 - -#Feature: Geo Enrichment -##Feature Description: Appends geo information about known non-local IPs to a telemetry message - -bolt.enrichment.geo.name=GeoEnrichment -bolt.enrichment.geo.enabled=true -bolt.enrichment.geo.num.tasks=1 -bolt.enrichment.geo.parallelism.hint=1 - -#Feature: Whois Enrichment -##Feature Description: Appends whois information about known domains to a telemetry message - -bolt.enrichment.whois.name=WhoisEnrichment -bolt.enrichment.whois.enabled=false -bolt.enrichment.whois.num.tasks=1 -bolt.enrichment.whois.parallelism.hint=1 - -#Feature: CIF Enrichment -##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message - -bolt.enrichment.cif.name=SIFBolt -bolt.enrichment.cif.enabled=false -bolt.enrichment.cif.num.tasks=1 -bolt.enrichment.cif.parallelism.hint=1 - -#Feature: Threat Enrichment -##Feature Description: Appends information from Threat intelligence feeds to a telemetry message - -bolt.enrichment.threat.name=ThreatBolt -bolt.enrichment.threat.enabled=false -bolt.enrichment.threat.num.tasks=1 -bolt.enrichment.threat.parallelism.hint=1 - -#Feature: Rules-Based Alerts -##Feature Description: Tags messages with rules-based alerts - -bolt.alerts.name=Alerts -bolt.alerts.enabled=true -bolt.alerts.num.tasks=1 -bolt.alerts.parallelism.hint=1 - -#Feature: Indexer -##Feature Description: Indexes telemetry messages in ElasticSearch or Solr - -bolt.indexing.name=IndexBolt -bolt.indexing.enabled=true -bolt.indexing.num.tasks=1 -bolt.indexing.parallelism.hint=1 - -#Feature: Alerts Indexer -##Feature Description: Indexes alert messages in ElasticSearch or Solr - -bolt.alerts.indexing.name=AlertIndexBolt -bolt.alerts.indexing.enabled=true -bolt.alerts.indexing.num.tasks=1 -bolt.alerts.indexing.parallelism.hint=1 - -#Feature: Error Indexer -##Feature Description: Indexes error messages in ElasticSearch or Solr - -bolt.error.indexing.name=ErrorIndexBolt -bolt.error.indexing.enabled=true -bolt.error.indexing.num.tasks=1 -bolt.error.indexing.parallelism.hint=1 - -#Feature: Kafka Bolt -##Feature Description: Writes telemetry messages back into a Kafka topic - -bolt.kafka.name=KafkaBolt -bolt.kafka.enabled=true -bolt.kafka.num.tasks=1 -bolt.kafka.parallelism.hint=1 - -#Feature: HDFS Bolt -##Feature Description: Writes telemetry messages into HDFS - -bolt.hdfs.name=HDFSBolt -bolt.hdfs.enabled=false -bolt.hdfs.num.tasks=1 -bolt.hdfs.parallelism.hint=1 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/metrics.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/metrics.conf deleted file mode 100644 index 1daef3d889..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/metrics.conf +++ /dev/null @@ -1,26 +0,0 @@ -#reporters -com.opensoc.metrics.reporter.graphite=true -com.opensoc.metrics.reporter.console=false -com.opensoc.metrics.reporter.jmx=false - -#Graphite Addresses - -com.opensoc.metrics.graphite.address=localhost -com.opensoc.metrics.graphite.port=2023 - -#TelemetryParserBolt -com.opensoc.metrics.TelemetryParserBolt.acks=true -com.opensoc.metrics.TelemetryParserBolt.emits=true -com.opensoc.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -com.opensoc.metrics.GenericEnrichmentBolt.acks=true -com.opensoc.metrics.GenericEnrichmentBolt.emits=true -com.opensoc.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -com.opensoc.metrics.TelemetryIndexingBolt.acks=true -com.opensoc.metrics.TelemetryIndexingBolt.emits=true -com.opensoc.metrics.TelemetryIndexingBolt.fails=true diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/topology.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/topology.conf deleted file mode 100644 index 7da2a4913e..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/topology.conf +++ /dev/null @@ -1,101 +0,0 @@ -include = ../../etc/env/environment_common.conf -include = ../../etc/env/es_connection.conf -include = ../../etc/env/hdfs_connection.conf -include = ../../etc/env/mysql_connection.conf -include = metrics.conf -include = features_enabled.conf - -#Global Properties - -debug.mode=true -local.mode=true -num.workers=1 - -#Standard 5-tuple fields - -source.ip=ip_src_addr -source.port=ip_src_port -dest.ip=ip_dst_addr -dest.port=ip_dst_port -protocol=protocol - -#Test Spout -spout.test.parallelism.repeat=false - -#Kafka Spout -spout.kafka.topic=lancope_raw - -#Parser Bolt -bolt.parser.adapter=com.opensoc.parsing.parsers.BasicLancopeParser - -#Host Enrichment - -bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.host.enrichment_tag=host - - -#GeoEnrichment - -bolt.enrichment.geo.enrichment_tag=geo -bolt.enrichment.geo.adapter.table=GEO -bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr - -#WhoisEnrichment - -bolt.enrichment.whois.hbase.table.name=whois -bolt.enrichment.whois.enrichment_tag=whois -bolt.enrichment.whois.fields=host -bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10 - -#CIF Enrichment -bolt.enrichment.cif.tablename=cif_table -bolt.enrichment.cif.fields.host=host -bolt.enrichment.cif.fields.email=email -bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr -bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.cif.enrichment_tag=cif - -#Threat Enrichment -bolt.enrichment.threat.tablename=threat_table -bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr -bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.threat.enrichment_tag=threat - -#Indexing Bolt -bolt.indexing.indexname=lancope_index -bolt.indexing.timestamp=yyyy.MM.ww -bolt.indexing.documentname=lancope_doc -bolt.indexing.bulk=200 -bolt.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Indexing Bolt -bolt.alerts.indexing.indexname=alert -bolt.alerts.indexing.timestamp=yyyy.MM.ww -bolt.alerts.indexing.documentname=lancope_alert -bolt.alerts.indexing.bulk=1 -bolt.alerts.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Error Indexing Bolt -bolt.error.indexing.indexname=error -bolt.error.indexing.timestamp=yyyy.MM -bolt.error.indexing.documentname=lancope_error -bolt.error.indexing.bulk=1 -bolt.error.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#HDFS Bolt -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://nn1:8020 -bolt.hdfs.wip.file.path=/lancope/wip -bolt.hdfs.finished.file.path=/lancope/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec - -#Kafka Bolt -bolt.kafka.topic=lancope_enriched \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/topology_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/topology_identifier.conf deleted file mode 100644 index a68084ef13..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/lancope/topology_identifier.conf +++ /dev/null @@ -1,4 +0,0 @@ -#Each topology must have a unique identifier. This setting is required - -topology.id=lancope -instance.id=L001 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/features_enabled.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/features_enabled.conf deleted file mode 100644 index 29ea06d21b..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/features_enabled.conf +++ /dev/null @@ -1,113 +0,0 @@ -#Enable and disable features for each topology - -#Feature: Test spout -##Feature Description: Reads telemetry from file and ingests it into topology. Used for testing or bulk loading the topology - -spout.test.name=TestSpout -spout.test.enabled=true -spout.test.num.tasks=1 -spout.test.parallelism.hint=1 - -#Feature: Kafka spout -##Feature Description: Acts as a Kafka consumer. Takes messages from a Kafka topic and ingests them into a topology - -spout.kafka.name=KafkaSpout -spout.kafka.enabled=false -spout.kafka.num.tasks=1 -spout.kafka.parallelism.hint=1 - -#Feature: Parser Bolt -##Feature Description: Parses telemetry from its native format into a native JSON - -bolt.parser.name=ParserBolt -bolt.parser.enabled=true -bolt.parser.num.tasks=1 -bolt.parser.parallelism.hint=1 - -#Feature: Host Enrichment -##Feature Description: Appends information about known hosts to a telemetry message - -bolt.enrichment.host.name=HostEnrichment -bolt.enrichment.host.enabled=true -bolt.enrichment.host.num.tasks=1 -bolt.enrichment.host.parallelism.hint=1 - -#Feature: Geo Enrichment -##Feature Description: Appends geo information about known non-local IPs to a telemetry message - -bolt.enrichment.geo.name=GeoEnrichment -bolt.enrichment.geo.enabled=true -bolt.enrichment.geo.num.tasks=1 -bolt.enrichment.geo.parallelism.hint=1 - -#Feature: Whois Enrichment -##Feature Description: Appends whois information about known domains to a telemetry message - -bolt.enrichment.whois.name=WhoisEnrichment -bolt.enrichment.whois.enabled=true -bolt.enrichment.whois.num.tasks=1 -bolt.enrichment.whois.parallelism.hint=1 - -#Feature: CIF Enrichment -##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message - -bolt.enrichment.cif.name=CIFBolt -bolt.enrichment.cif.enabled=true -bolt.enrichment.cif.num.tasks=1 -bolt.enrichment.cif.parallelism.hint=1 - -#Feature: Threat Enrichment -##Feature Description: Appends information from Threat intelligence feeds to a telemetry message - -bolt.enrichment.threat.name=ThreatBolt -bolt.enrichment.threat.enabled=false -bolt.enrichment.threat.num.tasks=1 -bolt.enrichment.threat.parallelism.hint=1 - -#Feature: Rules-Based Alerts -##Feature Description: Tags messages with rules-based alerts - -bolt.alerts.name=Alerts -bolt.alerts.enabled=true -bolt.alerts.num.tasks=1 -bolt.alerts.parallelism.hint=1 - -#Feature: Indexer -##Feature Description: Indexes telemetry messages in ElasticSearch or Solr - -bolt.indexing.name=IndexBolt -bolt.indexing.enabled=true -bolt.indexing.num.tasks=1 -bolt.indexing.parallelism.hint=1 - -#Feature: Alerts Indexer -##Feature Description: Indexes alert messages in ElasticSearch or Solr - -bolt.alerts.indexing.name=AlertIndexBolt -bolt.alerts.indexing.enabled=true -bolt.alerts.indexing.num.tasks=1 -bolt.alerts.indexing.parallelism.hint=1 - -#Feature: Error Indexer -##Feature Description: Indexes error messages in ElasticSearch or Solr - -bolt.error.indexing.name=ErrorIndexBolt -bolt.error.indexing.enabled=true -bolt.error.indexing.num.tasks=1 -bolt.error.indexing.parallelism.hint=1 - -#Feature: Kafka Bolt -##Feature Description: Writes telemetry messages back into a Kafka topic - -bolt.kafka.name=KafkaBolt -bolt.kafka.enabled=false -bolt.kafka.num.tasks=1 -bolt.kafka.parallelism.hint=1 - -#Feature: HDFS Bolt -##Feature Description: Writes telemetry messages into HDFS - -bolt.hdfs.name=HDFSBolt -bolt.hdfs.enabled=false -bolt.hdfs.num.tasks=1 -bolt.hdfs.parallelism.hint=1 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/metrics.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/metrics.conf deleted file mode 100644 index 1daef3d889..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/metrics.conf +++ /dev/null @@ -1,26 +0,0 @@ -#reporters -com.opensoc.metrics.reporter.graphite=true -com.opensoc.metrics.reporter.console=false -com.opensoc.metrics.reporter.jmx=false - -#Graphite Addresses - -com.opensoc.metrics.graphite.address=localhost -com.opensoc.metrics.graphite.port=2023 - -#TelemetryParserBolt -com.opensoc.metrics.TelemetryParserBolt.acks=true -com.opensoc.metrics.TelemetryParserBolt.emits=true -com.opensoc.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -com.opensoc.metrics.GenericEnrichmentBolt.acks=true -com.opensoc.metrics.GenericEnrichmentBolt.emits=true -com.opensoc.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -com.opensoc.metrics.TelemetryIndexingBolt.acks=true -com.opensoc.metrics.TelemetryIndexingBolt.emits=true -com.opensoc.metrics.TelemetryIndexingBolt.fails=true diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/topology.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/topology.conf deleted file mode 100644 index a92c7f3800..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/topology.conf +++ /dev/null @@ -1,113 +0,0 @@ -include = ../../etc/env/environment_common.conf -include = ../../etc/env/es_connection.conf -include = ../../etc/env/hdfs_connection.conf -include = ../../etc/env/mysql_connection.conf -include = metrics.conf -include = features_enabled.conf - -#Global Properties - -debug.mode=true -local.mode=true -num.workers=1 - -#Standard 5-tuple fields - -source.ip=ip_src_addr -source.port=ip_src_port -dest.ip=ip_dst_addr -dest.port=ip_dst_port -protocol=protocol - -#Test Spout -spout.test.parallelism.repeat=false - -#Kafka Spout -spout.kafka.topic=paloalto_raw - -#Parser Bolt -bolt.parser.adapter=com.opensoc.parsing.parsers.BasicPaloAltoFirewallParser - -#Host Enrichment - -bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.host.enrichment_tag=host - - -#GeoEnrichment - -bolt.enrichment.geo.enrichment_tag=geo -bolt.enrichment.geo.adapter.table=GEO -bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr - -#WhoisEnrichment - -bolt.enrichment.whois.hbase.table.name=whois -bolt.enrichment.whois.enrichment_tag=whois -bolt.enrichment.whois.fields=host -bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.whois.fields=host - -#CIF Enrichment -bolt.enrichment.cif.tablename=cif_table -bolt.enrichment.cif.fields.host=host -bolt.enrichment.cif.fields.email=email -bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr -bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.cif.enrichment_tag=cif -bolt.enrichment.cif.host=host - - -#Threat Enrichment -bolt.enrichment.threat.tablename=threat_table -bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr -bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.threat.enrichment_tag=threat - -#Indexing Bolt -bolt.indexing.indexname=paloalto_index -bolt.indexing.timestamp=yyyy.MM.ww -bolt.indexing.documentname=paloalto_doc -bolt.indexing.bulk=1 -bolt.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Indexing Bolt -bolt.alerts.indexing.indexname=alert -bolt.alerts.indexing.timestamp=yyyy.MM.ww -bolt.alerts.indexing.documentname=paloalto_alert -bolt.alerts.indexing.bulk=1 -bolt.alerts.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Error Indexing Bolt -bolt.error.indexing.indexname=error -bolt.error.indexing.timestamp.yyyy.MM -bolt.error.indexing.documentname=paloalto_error -bolt.error.indexing.bulk=1 -bolt.error.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Bolt -bolt.alerts.adapter=com.opensoc.alerts.adapters.CIFAlertsAdapter -com.opensoc.alerts.adapters.CIFAlertsAdapter.whitelist_table_name = ip_whitelist -com.opensoc.alerts.adapters.CIFAlertsAdapter.blacklist_table_name = ip_blacklist -com.opensoc.alerts.adapters.CIFAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3 -com.opensoc.alerts.adapters.CIFAlertsAdapter.port=2181 -com.opensoc.alerts.adapters.CIFAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600 -com.opensoc.alerts.adapters.CIFAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000 - -#HDFS Bolt -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://nn1:8020 -bolt.hdfs.wip.file.path=/paloalto/wip -bolt.hdfs.finished.file.path=/paloalto/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec - -#Kafka Bolt -bolt.kafka.topic=paloalto_enriched \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/topology_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/topology_identifier.conf deleted file mode 100644 index 7601122d25..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/paloalto/topology_identifier.conf +++ /dev/null @@ -1,4 +0,0 @@ -#Each topology must have a unique identifier. This setting is required - -topology.id=paloalto -instance.id=PA001 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/features_enabled.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/features_enabled.conf deleted file mode 100644 index 9b41fa2741..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/features_enabled.conf +++ /dev/null @@ -1,118 +0,0 @@ -#Enable and disable features for each topology - -#Feature: Test spout -##Feature Description: Reads telemetry from file and ingests it into topology. Used for testing or bulk loading the topology - -spout.test.name=TestSpout -spout.test.enabled=true -spout.test.num.tasks=1 -spout.test.parallelism.hint=1 - -#Feature: Kafka spout -##Feature Description: Acts as a Kafka consumer. Takes messages from a Kafka topic and ingests them into a topology - -spout.kafka.name=KafkaSpout -spout.kafka.enabled=false -spout.kafka.num.tasks=1 -spout.kafka.parallelism.hint=1 - -#Feature: Parser Bolt -##Feature Description: Parses telemetry from its native format into a native JSON - -bolt.parser.name=ParserBolt -bolt.parser.enabled=true -bolt.parser.num.tasks=1 -bolt.parser.parallelism.hint=1 - -#Feature: Host Enrichment -##Feature Description: Appends information about known hosts to a telemetry message - -bolt.enrichment.host.name=HostEnrichment -bolt.enrichment.host.enabled=false -bolt.enrichment.host.num.tasks=1 -bolt.enrichment.host.parallelism.hint=1 - -#Feature: Geo Enrichment -##Feature Description: Appends geo information about known non-local IPs to a telemetry message - -bolt.enrichment.geo.name=GeoEnrichment -bolt.enrichment.geo.enabled=false -bolt.enrichment.geo.num.tasks=1 -bolt.enrichment.geo.parallelism.hint=1 - -#Feature: Whois Enrichment -##Feature Description: Appends whois information about known domains to a telemetry message - -bolt.enrichment.whois.name=WhoisEnrichment -bolt.enrichment.whois.enabled=false -bolt.enrichment.whois.num.tasks=1 -bolt.enrichment.whois.parallelism.hint=1 - -#Feature: CIF Enrichment -##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message - -bolt.enrichment.cif.name=SIFBolt -bolt.enrichment.cif.enabled=false -bolt.enrichment.cif.num.tasks=1 -bolt.enrichment.cif.parallelism.hint=1 - -#Feature: Threat Enrichment -##Feature Description: Appends information from Threat intelligence feeds to a telemetry message - -bolt.enrichment.threat.name=ThreatBolt -bolt.enrichment.threat.enabled=false -bolt.enrichment.threat.num.tasks=1 -bolt.enrichment.threat.parallelism.hint=1 - -#Feature: Rules-Based Alerts -##Feature Description: Tags messages with rules-based alerts - -bolt.alerts.name=Alerts -bolt.alerts.enabled=false -bolt.alerts.num.tasks=1 -bolt.alerts.parallelism.hint=1 - -#Feature: Indexer -##Feature Description: Indexes telemetry messages in ElasticSearch or Solr - -bolt.indexing.name=IndexBolt -bolt.indexing.enabled=true -bolt.indexing.num.tasks=1 -bolt.indexing.parallelism.hint=1 - -#Feature: Alerts Indexer -##Feature Description: Indexes alert messages in ElasticSearch or Solr - -bolt.alerts.indexing.name=AlertIndexBolt -bolt.alerts.indexing.enabled=false -bolt.alerts.indexing.num.tasks=1 -bolt.alerts.indexing.parallelism.hint=1 - -#Feature: Error Indexer -##Feature Description: Indexes error messages in ElasticSearch or Solr - -bolt.error.indexing.name=ErrorIndexBolt -bolt.error.indexing.enabled=true -bolt.error.indexing.num.tasks=1 -bolt.error.indexing.parallelism.hint=1 - -#Feature: Kafka Bolt -##Feature Description: Writes telemetry messages back into a Kafka topic - -bolt.kafka.name=KafkaBolt -bolt.kafka.enabled=false -bolt.kafka.num.tasks=1 -bolt.kafka.parallelism.hint=1 - -#Feature: HDFS Bolt -##Feature Description: Writes telemetry messages into HDFS - -bolt.hdfs.name=HDFSBolt -bolt.hdfs.enabled=true -bolt.hdfs.num.tasks=1 -bolt.hdfs.parallelism.hint=1 - -bolt.hbase.name=HBaseBolt -bolt.hbase.enabled=true -bolt.hbase.num.tasks=1 -bolt.hbase.parallelism.hint=1 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/metrics.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/metrics.conf deleted file mode 100644 index 1daef3d889..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/metrics.conf +++ /dev/null @@ -1,26 +0,0 @@ -#reporters -com.opensoc.metrics.reporter.graphite=true -com.opensoc.metrics.reporter.console=false -com.opensoc.metrics.reporter.jmx=false - -#Graphite Addresses - -com.opensoc.metrics.graphite.address=localhost -com.opensoc.metrics.graphite.port=2023 - -#TelemetryParserBolt -com.opensoc.metrics.TelemetryParserBolt.acks=true -com.opensoc.metrics.TelemetryParserBolt.emits=true -com.opensoc.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -com.opensoc.metrics.GenericEnrichmentBolt.acks=true -com.opensoc.metrics.GenericEnrichmentBolt.emits=true -com.opensoc.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -com.opensoc.metrics.TelemetryIndexingBolt.acks=true -com.opensoc.metrics.TelemetryIndexingBolt.emits=true -com.opensoc.metrics.TelemetryIndexingBolt.fails=true diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/topology.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/topology.conf deleted file mode 100644 index 30c3ef34c2..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/topology.conf +++ /dev/null @@ -1,150 +0,0 @@ -include = ../../etc/env/environment_common.conf -include = ../../etc/env/es_connection.conf -include = ../../etc/env/hdfs_connection.conf -include = ../../etc/env/mysql_connection.conf -include = metrics.conf -include = features_enabled.conf - -#Global Properties - -debug.mode=true -local.mode=true -num.workers=1 - -#Standard 5-tuple fields - -source.ip=ip_src_addr -source.port=ip_src_port -dest.ip=ip_dst_addr -dest.port=ip_dst_port -protocol=protocol - -#Kafka Spout -spout.kafka.buffer.size.bytes=1024000 -spout.kafka.consumer.id=pcap.kafka -spout.kafka.fetch.size.bytes=1024 -spout.kafka.forcefromstart=false -spout.kafka.socket.timeout.ms=600000 -spout.kafka.start.offset.time=-1 -spout.kafka.zk.root=/storm/topology/pcap/kafka -spout.kafka.topic=pcap - -#Parser Bolt -bolt.parser.enabled=true -bolt.parser.num.of.key.chars.to.use.for.shuffle.grouping=6 -bolt.parser.ts.precision=MICRO - -#Test Spout -spout.test.parallelism.repeat=false - -#Kafka Spout -spout.kafka.topic=pcap_raw - - - -#Host Enrichment - -bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.host.enrichment_tag=host - - -#GeoEnrichment - -bolt.enrichment.geo.enrichment_tag=geo -bolt.enrichment.geo.adapter.table=GEO -bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.geo.source=ip_src_addr,ip_dst_addr - -#WhoisEnrichment - -bolt.enrichment.whois.hbase.table.name=whois -bolt.enrichment.whois.enrichment_tag=whois -bolt.enrichment.whois.fields=host -bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10 - -#CIF Enrichment -bolt.enrichment.cif.tablename=cif_table -bolt.enrichment.cif.fields.host=host -bolt.enrichment.cif.fields.email=email -bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.cif.enrichment_tag=cif -bolt.enrichment.cif.ip=ip_src_addr,_ip_dst_addr - -#Threat Enrichment -bolt.enrichment.threat.tablename=threat_table -bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr -bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.threat.enrichment_tag=threat - -#Indexing Bolt -bolt.indexing.indexname=pcap_index -bolt.indexing.timestamp=yyyy.MM.dd.hh -bolt.indexing.documentname=pcap_doc -bolt.indexing.bulk=1 -bolt.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Indexing Bolt -bolt.alerts.indexing.indexname=pcap_alert_test -bolt.alerts.indexing.documentname=pcap_alert -bolt.alerts.indexing.bulk=1 -bolt.alerts.indexing.adapter=com.opensoc.indexing.adapters.ESBaseBulkAdapter - -#Error Indexing Bolt -bolt.error.indexing.indexname=error -bolt.error.indexing.timestamp=yyyy.MM -bolt.error.indexing.documentname=pcap_error -bolt.error.indexing.bulk=1 -bolt.error.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#HDFS Bolt -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://nn1:8020 -bolt.hdfs.wip.file.path=/pcap/wip -bolt.hdfs.finished.file.path=/pcap/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec - -#Kafka Bolt -bolt.kafka.topic=pcap_enriched - -#HBase Bolt -bolt.hbase.table.name=pcap_test -## Define the hbase table columns in the form :,,|:,|....... -bolt.hbase.table.fields=t:pcap -bolt.hbase.table.key.tuple.field.name=pcap_id -bolt.hbase.table.timestamp.tuple.field.name=timestamp -bolt.hbase.enable.batching=false -bolt.hbase.write.buffer.size.in.bytes=2000000 -bolt.hbase.durability=SKIP_WAL -bolt.hbase.partitioner.region.info.refresh.interval.mins=60 - - -#Extra [Optional] Storm Configuration Options - -optional.settings.bolt.index.search.transport.tcp.compress=true -optional.settings.bolt.index.search.discovery.zen.ping.multicast.enabled:true -optional.settings.bolt.index.search.discovery.zen.ping.unicast.hosts=ctrl01:9300\,ctrl02:9300\,ctrl03:9300 -optional.settings.bolt.index.search.http.port=19200 -optional.settings.bolt.index.search.transport.tcp.port=19300 -optional.settings.bolt.index.search.node.name=node.name_{index} -optional.settings.bolt.index.search.path.data=/tmp/es_data_client_{index} -optional.settings.bolt.index.search.path.work=/tmp/es_work_client_{index} -optional.settings.bolt.index.search.path.logs=/var/log/elasticsearch/client_{index} -optional.settings.bolt.index.search.http.enabled=true -optional.settings.bolt.index.search.discovery.zen.minimum_master_nodes=1 -optional.settings.bolt.index.search.discovery.zen.ping.multicast.ttl=60 -optional.settings.bolt.index.search.discovery.zen.ping_timeout=500 -optional.settings.bolt.index.search.discovery.zen.fd.ping_timeout=500 -optional.settings.bolt.index.search.discovery.zen.fd.ping_interval=60 -optional.settings.bolt.index.search.discovery.zen.fd.ping_retries=60 -optional.settings.bolt.index.search.client.transport.ping_timeout=60s -optional.settings.bolt.index.search.multicast.enabled=false -optional.settings.bolt.index.search.index.refresh_interval=2m -optional.settings.bolt.index.search.index.merge.async=true -optional.settings.bolt.index.search.action.write_consistency=one diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/topology_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/topology_identifier.conf deleted file mode 100644 index aad3257fc6..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/pcap/topology_identifier.conf +++ /dev/null @@ -1,4 +0,0 @@ -#Each topology must have a unique identifier. This setting is required - -topology.id=pcap -instance.id=P001 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/alerts.xml b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/alerts.xml deleted file mode 100644 index 9286b10df4..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/alerts.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - .*message.* - {"type":"alert","priority":5, "title":"Sourcefire Alert", "body": - "Alert triggered by sourcefire"} - - - - - diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/features_enabled.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/features_enabled.conf deleted file mode 100644 index 5b45ddef9a..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/features_enabled.conf +++ /dev/null @@ -1,113 +0,0 @@ -#Enable and disable features for each topology - -#Feature: Test spout -##Feature Description: Reads telemetry from file and ingests it into topology. Used for testing or bulk loading the topology - -spout.test.name=TestSpout -spout.test.enabled=true -spout.test.num.tasks=1 -spout.test.parallelism.hint=1 - -#Feature: Kafka spout -##Feature Description: Acts as a Kafka consumer. Takes messages from a Kafka topic and ingests them into a topology - -spout.kafka.name=KafkaSpout -spout.kafka.enabled=false -spout.kafka.num.tasks=1 -spout.kafka.parallelism.hint=1 - -#Feature: Parser Bolt -##Feature Description: Parses telemetry from its native format into a native JSON - -bolt.parser.name=ParserBolt -bolt.parser.enabled=true -bolt.parser.num.tasks=1 -bolt.parser.parallelism.hint=1 - -#Feature: Host Enrichment -##Feature Description: Appends information about known hosts to a telemetry message - -bolt.enrichment.host.name=HostEnrichment -bolt.enrichment.host.enabled=true -bolt.enrichment.host.num.tasks=1 -bolt.enrichment.host.parallelism.hint=1 - -#Feature: Geo Enrichment -##Feature Description: Appends geo information about known non-local IPs to a telemetry message - -bolt.enrichment.geo.name=GeoEnrichment -bolt.enrichment.geo.enabled=true -bolt.enrichment.geo.num.tasks=1 -bolt.enrichment.geo.parallelism.hint=1 - -#Feature: Whois Enrichment -##Feature Description: Appends whois information about known domains to a telemetry message - -bolt.enrichment.whois.name=WhoisEnrichment -bolt.enrichment.whois.enabled=false -bolt.enrichment.whois.num.tasks=1 -bolt.enrichment.whois.parallelism.hint=1 - -#Feature: CIF Enrichment -##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message - -bolt.enrichment.cif.name=SIFBolt -bolt.enrichment.cif.enabled=false -bolt.enrichment.cif.num.tasks=1 -bolt.enrichment.cif.parallelism.hint=1 - -#Feature: Threat Enrichment -##Feature Description: Appends information from Threat intelligence feeds to a telemetry message - -bolt.enrichment.threat.name=ThreatBolt -bolt.enrichment.threat.enabled=false -bolt.enrichment.threat.num.tasks=1 -bolt.enrichment.threat.parallelism.hint=1 - -#Feature: Rules-Based Alerts -##Feature Description: Tags messages with rules-based alerts - -bolt.alerts.name=Alerts -bolt.alerts.enabled=true -bolt.alerts.num.tasks=1 -bolt.alerts.parallelism.hint=1 - -#Feature: Indexer -##Feature Description: Indexes telemetry messages in ElasticSearch or Solr - -bolt.indexing.name=IndexBolt -bolt.indexing.enabled=true -bolt.indexing.num.tasks=1 -bolt.indexing.parallelism.hint=1 - -#Feature: Alerts Indexer -##Feature Description: Indexes alert messages in ElasticSearch or Solr - -bolt.alerts.indexing.name=AlertIndexBolt -bolt.alerts.indexing.enabled=true -bolt.alerts.indexing.num.tasks=1 -bolt.alerts.indexing.parallelism.hint=1 - -#Feature: Error Indexer -##Feature Description: Indexes error messages in ElasticSearch or Solr - -bolt.error.indexing.name=ErrorIndexBolt -bolt.error.indexing.enabled=true -bolt.error.indexing.num.tasks=1 -bolt.error.indexing.parallelism.hint=1 - -#Feature: Kafka Bolt -##Feature Description: Writes telemetry messages back into a Kafka topic - -bolt.kafka.name=KafkaBolt -bolt.kafka.enabled=false -bolt.kafka.num.tasks=1 -bolt.kafka.parallelism.hint=1 - -#Feature: HDFS Bolt -##Feature Description: Writes telemetry messages into HDFS - -bolt.hdfs.name=HDFSBolt -bolt.hdfs.enabled=false -bolt.hdfs.num.tasks=1 -bolt.hdfs.parallelism.hint=1 \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/metrics.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/metrics.conf deleted file mode 100644 index 1daef3d889..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/metrics.conf +++ /dev/null @@ -1,26 +0,0 @@ -#reporters -com.opensoc.metrics.reporter.graphite=true -com.opensoc.metrics.reporter.console=false -com.opensoc.metrics.reporter.jmx=false - -#Graphite Addresses - -com.opensoc.metrics.graphite.address=localhost -com.opensoc.metrics.graphite.port=2023 - -#TelemetryParserBolt -com.opensoc.metrics.TelemetryParserBolt.acks=true -com.opensoc.metrics.TelemetryParserBolt.emits=true -com.opensoc.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -com.opensoc.metrics.GenericEnrichmentBolt.acks=true -com.opensoc.metrics.GenericEnrichmentBolt.emits=true -com.opensoc.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -com.opensoc.metrics.TelemetryIndexingBolt.acks=true -com.opensoc.metrics.TelemetryIndexingBolt.emits=true -com.opensoc.metrics.TelemetryIndexingBolt.fails=true diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/topology.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/topology.conf deleted file mode 100644 index 29d682aec4..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/topology.conf +++ /dev/null @@ -1,110 +0,0 @@ -include = ../../etc/env/environment_common.conf -include = ../../etc/env/es_connection.conf -include = ../../etc/env/hdfs_connection.conf -include = ../../etc/env/mysql_connection.conf -include = metrics.conf -include = features_enabled.conf - -#Global Properties - -debug.mode=true -local.mode=true -num.workers=1 - -#Standard 5-tuple fields - -source.ip=ip_src_addr -source.port=ip_src_port -dest.ip=ip_dst_addr -dest.port=ip_dst_port -protocol=protocol - -#Test Spout -spout.test.parallelism.repeat=false - -#Kafka Spout -spout.kafka.topic=sourcefire_raw - -#Parser Bolt -bolt.parser.adapter=com.opensoc.parsing.parsers.BasicSourcefireParser - -#Host Enrichment - -bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.host.enrichment_tag=host - - -#GeoEnrichment - -bolt.enrichment.geo.enrichment_tag=geo -bolt.enrichment.geo.adapter.table=GEO -bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr - -#WhoisEnrichment - -bolt.enrichment.whois.hbase.table.name=whois -bolt.enrichment.whois.enrichment_tag=whois -bolt.enrichment.whois.fields=host -bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10 - -#CIF Enrichment -bolt.enrichment.cif.tablename=cif_table -bolt.enrichment.cif.fields.host=host -bolt.enrichment.cif.fields.email=email -bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr -bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.cif.enrichment_tag=cif - -#Threat Enrichment -bolt.enrichment.threat.tablename=threat_table -bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr -bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000 -bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10 -bolt.enrichment.threat.enrichment_tag=threat - -#Indexing Bolt -bolt.indexing.indexname=sourcefire_index -bolt.indexing.timestamp=yyyy.MM.ww -bolt.indexing.documentname=sourcefire_doc -bolt.indexing.bulk=1 -bolt.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Indexing Bolt -bolt.alerts.indexing.indexname=alert -bolt.alerts.indexing.timestamp=yyyy.MM.ww -bolt.alerts.indexing.documentname=sourcefire_alert -bolt.alerts.indexing.bulk=1 -bolt.alerts.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Error Indexing Bolt -bolt.error.indexing.indexname=error -bolt.error.indexing.timestamp=yyyy.MM -bolt.error.indexing.documentname=sourcefire_error -bolt.error.indexing.bulk=1 -bolt.error.indexing.adapter=com.opensoc.indexing.adapters.ESTimedRotatingAdapter - -#Alerts Bolt -bolt.alerts.adapter=com.opensoc.alerts.adapters.AllAlertAdapter -com.opensoc.alerts.adapters.AllAlertAdapter.whitelist_table_name = ip_whitelist -com.opensoc.alerts.adapters.AllAlertAdapter.blacklist_table_name = ip_blacklist -com.opensoc.alerts.adapters.AllAlertAdapter.quorum=zkpr1,zkpr2,zkpr3 -com.opensoc.alerts.adapters.AllAlertAdapter.port=2181 -com.opensoc.alerts.adapters.AllAlertAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600 -com.opensoc.alerts.adapters.AllAlertAdapter._MAX_TIME_RETAIN_MINUTES=1000 - -#HDFS Bolt -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://nn1:8020 -bolt.hdfs.wip.file.path=/sourcefire/wip -bolt.hdfs.finished.file.path=/sourcefire/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec - -#Kafka Bolt -bolt.kafka.topic=sourcefire_enriched \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/topology_identifier.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/topology_identifier.conf deleted file mode 100644 index da85baefcf..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/OpenSOC_Configs/topologies/sourcefire/topology_identifier.conf +++ /dev/null @@ -1,4 +0,0 @@ -#Each topology must have a unique identifier. This setting is required - -topology.id=sourcefire -instance.id=S001 \ No newline at end of file From 5b3a14b750e5e416913c0bb13bfe2fde1675daf5 Mon Sep 17 00:00:00 2001 From: Sean Schulte Date: Wed, 13 Jan 2016 11:17:49 -0600 Subject: [PATCH 6/7] Removed old topology runner/builder Java code. --- .../main/java/com/opensoc/topology/Asa.java | 40 - .../main/java/com/opensoc/topology/Bro.java | 36 - .../java/com/opensoc/topology/FireEye.java | 21 - .../main/java/com/opensoc/topology/Ise.java | 39 - .../java/com/opensoc/topology/Lancope.java | 40 - .../opensoc/topology/PaloAltoFirewall.java | 41 - .../main/java/com/opensoc/topology/Pcap.java | 41 - .../java/com/opensoc/topology/Sourcefire.java | 40 - .../opensoc/topology/runner/AsaRunner.java | 94 -- .../opensoc/topology/runner/BroRunner.java | 89 -- .../topology/runner/FireEyeRunner.java | 77 -- .../opensoc/topology/runner/ISERunner.java | 91 -- .../topology/runner/LancopeRunner.java | 94 -- .../runner/PaloAltoFirewallRunner.java | 95 -- .../opensoc/topology/runner/PcapRunner.java | 72 -- .../topology/runner/SourcefireRunner.java | 94 -- .../topology/runner/TopologyRunner.java | 1047 ----------------- 17 files changed, 2051 deletions(-) delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Asa.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Bro.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/FireEye.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Ise.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Lancope.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/PaloAltoFirewall.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Pcap.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Sourcefire.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/AsaRunner.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/BroRunner.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/FireEyeRunner.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/ISERunner.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/LancopeRunner.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/PaloAltoFirewallRunner.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/PcapRunner.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/SourcefireRunner.java delete mode 100644 opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/TopologyRunner.java diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Asa.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Asa.java deleted file mode 100644 index 68f0c89e4b..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Asa.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology; - -import org.apache.commons.configuration.ConfigurationException; - -import backtype.storm.generated.InvalidTopologyException; - -import com.opensoc.topology.runner.AsaRunner; -import com.opensoc.topology.runner.TopologyRunner; - - -/** - * Topology for processing Asa messages - * - */ -public class Asa{ - - public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException { - - TopologyRunner runner = new AsaRunner(); - runner.initTopology(args, "asa"); - } - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Bro.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Bro.java deleted file mode 100644 index 280738c6ad..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Bro.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.opensoc.topology; - -import org.apache.commons.configuration.ConfigurationException; - -import backtype.storm.generated.InvalidTopologyException; - -import com.opensoc.topology.runner.BroRunner; -import com.opensoc.topology.runner.TopologyRunner; - -public class Bro{ - - public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException { - - TopologyRunner runner = new BroRunner(); - runner.initTopology(args, "bro"); - } - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/FireEye.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/FireEye.java deleted file mode 100644 index e1f489befc..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/FireEye.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.opensoc.topology; - -import org.apache.commons.configuration.ConfigurationException; -import backtype.storm.generated.InvalidTopologyException; -import com.opensoc.topology.runner.FireEyeRunner; -import com.opensoc.topology.runner.TopologyRunner; - - -/** - * Topology for processing FireEye syslog messages - * - */ -public class FireEye { - - public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException { - - TopologyRunner runner = new FireEyeRunner(); - runner.initTopology(args, "fireeye"); - } - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Ise.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Ise.java deleted file mode 100644 index 7bcd0c2811..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Ise.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology; - -import org.apache.commons.configuration.ConfigurationException; - -import backtype.storm.generated.InvalidTopologyException; - -import com.opensoc.topology.runner.ISERunner; -import com.opensoc.topology.runner.TopologyRunner; - -/** - * Topology for processing Ise messages - * - */ -public class Ise{ - - public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException { - - TopologyRunner runner = new ISERunner(); - runner.initTopology(args, "ise"); - } - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Lancope.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Lancope.java deleted file mode 100644 index c3ecc54beb..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Lancope.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology; - -import org.apache.commons.configuration.ConfigurationException; - -import backtype.storm.generated.InvalidTopologyException; - -import com.opensoc.topology.runner.LancopeRunner; -import com.opensoc.topology.runner.TopologyRunner; - - -/** - * Topology for processing Lancope messages - * - */ -public class Lancope{ - - public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException { - - TopologyRunner runner = new LancopeRunner(); - runner.initTopology(args, "lancope"); - } - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/PaloAltoFirewall.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/PaloAltoFirewall.java deleted file mode 100644 index 222cc29854..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/PaloAltoFirewall.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology; - -import org.apache.commons.configuration.ConfigurationException; - -import backtype.storm.generated.InvalidTopologyException; - -import com.opensoc.topology.runner.AsaRunner; -import com.opensoc.topology.runner.PaloAltoFirewallRunner; -import com.opensoc.topology.runner.TopologyRunner; - - -/** - * Topology for processing Palo Alto Firewall Syslog messages - * - */ -public class PaloAltoFirewall { - - public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException { - - TopologyRunner runner = new PaloAltoFirewallRunner(); - runner.initTopology(args, "paloalto"); - } - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Pcap.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Pcap.java deleted file mode 100644 index 25328931fc..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Pcap.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology; - -import org.apache.commons.configuration.ConfigurationException; - -import backtype.storm.generated.InvalidTopologyException; - -import com.opensoc.topology.runner.PcapRunner; -import com.opensoc.topology.runner.TopologyRunner; - - -/** - * Topology for processing raw packet messages - * - */ - -public class Pcap{ - - public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException { - - TopologyRunner runner = new PcapRunner(); - runner.initTopology(args, "pcap"); - } - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Sourcefire.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Sourcefire.java deleted file mode 100644 index bb8a43f53b..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/Sourcefire.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology; - -import org.apache.commons.configuration.ConfigurationException; - -import com.opensoc.topology.runner.SourcefireRunner; -import com.opensoc.topology.runner.TopologyRunner; - -import backtype.storm.generated.InvalidTopologyException; - - -/** - * Topology for processing Sourcefire messages - * - */ -public class Sourcefire{ - - public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException { - - TopologyRunner runner = new SourcefireRunner(); - runner.initTopology(args, "sourcefire"); - } - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/AsaRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/AsaRunner.java deleted file mode 100644 index 8cc2db7479..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/AsaRunner.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology.runner; - -import com.opensoc.filters.GenericMessageFilter; -import com.opensoc.parser.interfaces.MessageParser; -import com.opensoc.parsing.AbstractParserBolt; -import com.opensoc.parsing.TelemetryParserBolt; -import com.opensoc.test.spouts.GenericInternalTestSpout; - -public class AsaRunner extends TopologyRunner{ - - static String test_file_path = "SampleInput/AsaOutput"; - - @Override - public boolean initializeParsingBolt(String topology_name, - String name) { - try { - - String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); - - - String class_name = config.getString("bolt.parser.adapter"); - - if(class_name == null) - { - System.out.println("[OpenSOC] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf"); - throw new Exception("Parser adapter not set"); - } - - Class loaded_class = Class.forName(class_name); - MessageParser parser = (MessageParser) loaded_class.newInstance(); - - - AbstractParserBolt parser_bolt = new TelemetryParserBolt() - .withMessageParser(parser) - .withOutputFieldName(topology_name) - .withMessageFilter(new GenericMessageFilter()) - .withMetricConfig(config); - - builder.setBolt(name, parser_bolt, - config.getInt("bolt.parser.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent) - .setNumTasks(config.getInt("bolt.parser.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - @Override - public boolean initializeTestingSpout(String name) { - try { - - System.out.println("[OpenSOC] Initializing Test Spout"); - - GenericInternalTestSpout testSpout = new GenericInternalTestSpout() - .withFilename(test_file_path).withRepeating( - config.getBoolean("spout.test.parallelism.repeat")); - - builder.setSpout(name, testSpout, - config.getInt("spout.test.parallelism.hint")).setNumTasks( - config.getInt("spout.test.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/BroRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/BroRunner.java deleted file mode 100644 index c44801743e..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/BroRunner.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology.runner; - -import com.opensoc.filters.GenericMessageFilter; -import com.opensoc.parser.interfaces.MessageParser; -import com.opensoc.parsing.AbstractParserBolt; -import com.opensoc.parsing.TelemetryParserBolt; -import com.opensoc.test.spouts.GenericInternalTestSpout; - -public class BroRunner extends TopologyRunner{ - - static String test_file_path = "SampleInput/BroExampleOutput"; - - @Override - public boolean initializeParsingBolt(String topology_name, - String name) { - try { - - String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); - - String class_name = config.getString("bolt.parser.adapter"); - - if(class_name == null) - { - System.out.println("[OpenSOC] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf"); - throw new Exception("Parser adapter not set"); - } - - Class loaded_class = Class.forName(class_name); - MessageParser parser = (MessageParser) loaded_class.newInstance(); - - AbstractParserBolt parser_bolt = new TelemetryParserBolt() - .withMessageParser(parser) - .withOutputFieldName(topology_name) - .withMessageFilter(new GenericMessageFilter()) - .withMetricConfig(config); - - builder.setBolt(name, parser_bolt, - config.getInt("bolt.parser.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent) - .setNumTasks(config.getInt("bolt.parser.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - @Override - public boolean initializeTestingSpout(String name) { - try { - - System.out.println("[OpenSOC] Initializing Test Spout"); - - GenericInternalTestSpout testSpout = new GenericInternalTestSpout() - .withFilename(test_file_path).withRepeating( - config.getBoolean("spout.test.parallelism.repeat")); - - builder.setSpout(name, testSpout, - config.getInt("spout.test.parallelism.hint")).setNumTasks( - config.getInt("spout.test.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/FireEyeRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/FireEyeRunner.java deleted file mode 100644 index 31026df39e..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/FireEyeRunner.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.opensoc.topology.runner; - -import com.opensoc.filters.GenericMessageFilter; -import com.opensoc.parser.interfaces.MessageParser; -import com.opensoc.parsing.AbstractParserBolt; -import com.opensoc.parsing.TelemetryParserBolt; -import com.opensoc.test.spouts.GenericInternalTestSpout; - -public class FireEyeRunner extends TopologyRunner{ - - static String test_file_path = "SampleInput/FireeyeExampleOutput"; - - @Override - public boolean initializeParsingBolt(String topology_name, - String name) { - try { - - String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); - - - String class_name = config.getString("bolt.parser.adapter"); - - if(class_name == null) - { - System.out.println("[OpenSOC] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf"); - throw new Exception("Parser adapter not set"); - } - - Class loaded_class = Class.forName(class_name); - MessageParser parser = (MessageParser) loaded_class.newInstance(); - - - AbstractParserBolt parser_bolt = new TelemetryParserBolt() - .withMessageParser(parser) - .withOutputFieldName(topology_name) - .withMessageFilter(new GenericMessageFilter()) - .withMetricConfig(config); - - builder.setBolt(name, parser_bolt, - config.getInt("bolt.parser.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent) - .setNumTasks(config.getInt("bolt.parser.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - @Override - public boolean initializeTestingSpout(String name) { - try { - - System.out.println("[OpenSOC] Initializing Test Spout"); - - GenericInternalTestSpout testSpout = new GenericInternalTestSpout() - .withFilename(test_file_path).withRepeating( - config.getBoolean("spout.test.parallelism.repeat")); - - builder.setSpout(name, testSpout, - config.getInt("spout.test.parallelism.hint")).setNumTasks( - config.getInt("spout.test.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - - -} \ No newline at end of file diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/ISERunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/ISERunner.java deleted file mode 100644 index 7f377d5b83..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/ISERunner.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology.runner; - -import com.opensoc.filters.GenericMessageFilter; -import com.opensoc.parser.interfaces.MessageParser; -import com.opensoc.parsing.AbstractParserBolt; -import com.opensoc.parsing.TelemetryParserBolt; -import com.opensoc.test.spouts.GenericInternalTestSpout; - -public class ISERunner extends TopologyRunner{ - - static String test_file_path = "SampleInput/ISESampleOutput"; - - @Override - public boolean initializeParsingBolt(String topology_name, - String name) { - try { - - String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); - - - String class_name = config.getString("bolt.parser.adapter"); - - if(class_name == null) - { - System.out.println("[OpenSOC] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf"); - throw new Exception("Parser adapter not set"); - } - - Class loaded_class = Class.forName(class_name); - MessageParser parser = (MessageParser) loaded_class.newInstance(); - - - AbstractParserBolt parser_bolt = new TelemetryParserBolt() - .withMessageParser(parser) - .withOutputFieldName(topology_name) - .withMessageFilter(new GenericMessageFilter()) - .withMetricConfig(config); - - builder.setBolt(name, parser_bolt, - config.getInt("bolt.parser.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent) - .setNumTasks(config.getInt("bolt.parser.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - @Override - public boolean initializeTestingSpout(String name) { - try { - - System.out.println("[OpenSOC] Initializing Test Spout"); - - GenericInternalTestSpout testSpout = new GenericInternalTestSpout() - .withFilename(test_file_path).withRepeating( - config.getBoolean("spout.test.parallelism.repeat")); - - builder.setSpout(name, testSpout, - config.getInt("spout.test.parallelism.hint")).setNumTasks( - config.getInt("spout.test.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/LancopeRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/LancopeRunner.java deleted file mode 100644 index 1031abffe4..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/LancopeRunner.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology.runner; - -import com.opensoc.filters.GenericMessageFilter; -import com.opensoc.parser.interfaces.MessageParser; -import com.opensoc.parsing.AbstractParserBolt; -import com.opensoc.parsing.TelemetryParserBolt; -import com.opensoc.parsing.parsers.BasicLancopeParser; -import com.opensoc.test.spouts.GenericInternalTestSpout; - -public class LancopeRunner extends TopologyRunner{ - - static String test_file_path = "SampleInput/LancopeExampleOutput"; - - @Override - public boolean initializeParsingBolt(String topology_name, - String name) { - try { - - String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); - - - String class_name = config.getString("bolt.parser.adapter"); - - if(class_name == null) - { - System.out.println("[OpenSOC] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf"); - throw new Exception("Parser adapter not set"); - } - - Class loaded_class = Class.forName(class_name); - MessageParser parser = (MessageParser) loaded_class.newInstance(); - - AbstractParserBolt parser_bolt = new TelemetryParserBolt() - .withMessageParser(parser) - .withOutputFieldName(topology_name) - .withMessageFilter(new GenericMessageFilter()) - .withMetricConfig(config); - - builder.setBolt(name, parser_bolt, - config.getInt("bolt.parser.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent) - .setNumTasks(config.getInt("bolt.parser.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - @Override - public boolean initializeTestingSpout(String name) { - try { - - System.out.println("[OpenSOC] Initializing Test Spout"); - - GenericInternalTestSpout testSpout = new GenericInternalTestSpout() - .withFilename(test_file_path).withRepeating( - config.getBoolean("spout.test.parallelism.repeat")); - - builder.setSpout(name, testSpout, - config.getInt("spout.test.parallelism.hint")).setNumTasks( - config.getInt("spout.test.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/PaloAltoFirewallRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/PaloAltoFirewallRunner.java deleted file mode 100644 index 0b6adadbf5..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/PaloAltoFirewallRunner.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology.runner; - -import com.opensoc.filters.GenericMessageFilter; -import com.opensoc.parser.interfaces.MessageParser; -import com.opensoc.parsing.AbstractParserBolt; -import com.opensoc.parsing.TelemetryParserBolt; -import com.opensoc.test.spouts.GenericInternalTestSpout; - -public class PaloAltoFirewallRunner extends TopologyRunner{ - - static String test_file_path = "SampleInput/PaloaltoOutput"; - - @Override - public boolean initializeParsingBolt(String topology_name, - String name) { - try { - - String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); - - - String class_name = config.getString("bolt.parser.adapter"); - - if(class_name == null) - { - System.out.println("[OpenSOC] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf"); - throw new Exception("Parser adapter not set"); - } - - Class loaded_class = Class.forName(class_name); - MessageParser parser = (MessageParser) loaded_class.newInstance(); - - - - AbstractParserBolt parser_bolt = new TelemetryParserBolt() - .withMessageParser(parser) - .withOutputFieldName(topology_name) - .withMessageFilter(new GenericMessageFilter()) - .withMetricConfig(config); - - builder.setBolt(name, parser_bolt, - config.getInt("bolt.parser.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent) - .setNumTasks(config.getInt("bolt.parser.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - @Override - public boolean initializeTestingSpout(String name) { - try { - - System.out.println("[OpenSOC] Initializing Test Spout"); - - GenericInternalTestSpout testSpout = new GenericInternalTestSpout() - .withFilename(test_file_path).withRepeating( - config.getBoolean("spout.test.parallelism.repeat")); - - builder.setSpout(name, testSpout, - config.getInt("spout.test.parallelism.hint")).setNumTasks( - config.getInt("spout.test.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/PcapRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/PcapRunner.java deleted file mode 100644 index a26a467e46..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/PcapRunner.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology.runner; - -import com.opensoc.parsing.PcapParserBolt; -import com.opensoc.test.spouts.GenericInternalTestSpout; - -public class PcapRunner extends TopologyRunner{ - - static String test_file_path = "SampleInput/PCAPExampleOutput"; - - - @Override - public boolean initializeTestingSpout(String name) { - try { - - - System.out.println("[OpenSOC] Initializing Test Spout"); - - - GenericInternalTestSpout testSpout = new GenericInternalTestSpout() - .withFilename(test_file_path).withRepeating( - config.getBoolean("spout.test.parallelism.repeat")); - - builder.setSpout(name, testSpout, - config.getInt("spout.test.parallelism.hint")).setNumTasks( - config.getInt("spout.test.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - @Override - boolean initializeParsingBolt(String topology_name, String name) { - try { - - String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); - - PcapParserBolt pcapParser = new PcapParserBolt().withTsPrecision(config.getString("bolt.parser.ts.precision")); - - builder.setBolt(name, pcapParser, - config.getInt("bolt.parser.parallelism.hint")) - .setNumTasks(config.getInt("bolt.parser.num.tasks")) - .shuffleGrouping(messageUpstreamComponent); - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/SourcefireRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/SourcefireRunner.java deleted file mode 100644 index 69b4581f74..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/SourcefireRunner.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology.runner; - -import com.opensoc.filters.GenericMessageFilter; -import com.opensoc.parser.interfaces.MessageParser; -import com.opensoc.parsing.AbstractParserBolt; -import com.opensoc.parsing.TelemetryParserBolt; -import com.opensoc.test.spouts.GenericInternalTestSpout; - -public class SourcefireRunner extends TopologyRunner{ - - static String test_file_path = "SampleInput/SourcefireExampleOutput"; - - @Override - public boolean initializeParsingBolt(String topology_name, - String name) { - try { - - String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); - - - String class_name = config.getString("bolt.parser.adapter"); - - if(class_name == null) - { - System.out.println("[OpenSOC] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf"); - throw new Exception("Parser adapter not set"); - } - - Class loaded_class = Class.forName(class_name); - MessageParser parser = (MessageParser) loaded_class.newInstance(); - - - AbstractParserBolt parser_bolt = new TelemetryParserBolt() - .withMessageParser(parser) - .withOutputFieldName(topology_name) - .withMessageFilter(new GenericMessageFilter()) - .withMetricConfig(config); - - builder.setBolt(name, parser_bolt, - config.getInt("bolt.parser.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent) - .setNumTasks(config.getInt("bolt.parser.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - @Override - public boolean initializeTestingSpout(String name) { - try { - - System.out.println("[OpenSOC] Initializing Test Spout"); - - GenericInternalTestSpout testSpout = new GenericInternalTestSpout() - .withFilename(test_file_path).withRepeating( - config.getBoolean("spout.test.parallelism.repeat")); - - builder.setSpout(name, testSpout, - config.getInt("spout.test.parallelism.hint")).setNumTasks( - config.getInt("spout.test.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - - -} diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/TopologyRunner.java b/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/TopologyRunner.java deleted file mode 100644 index ed3282539f..0000000000 --- a/opensoc-streaming/OpenSOC-Topologies/src/main/java/com/opensoc/topology/runner/TopologyRunner.java +++ /dev/null @@ -1,1047 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opensoc.topology.runner; - -import java.lang.reflect.Constructor; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; - -import oi.thekraken.grok.api.Grok; - -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.lang.StringUtils; -import org.apache.storm.hdfs.bolt.HdfsBolt; -import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat; -import org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat; -import org.apache.storm.hdfs.bolt.format.FileNameFormat; -import org.apache.storm.hdfs.bolt.format.RecordFormat; -import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy; -import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy; -import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units; -import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy; -import org.apache.storm.hdfs.bolt.sync.SyncPolicy; -import org.apache.storm.hdfs.common.rotation.MoveFileAction; -import org.json.simple.JSONObject; - -import storm.kafka.BrokerHosts; -import storm.kafka.KafkaSpout; -import storm.kafka.SpoutConfig; -import storm.kafka.ZkHosts; -import storm.kafka.bolt.KafkaBolt; -import backtype.storm.Config; -import backtype.storm.LocalCluster; -import backtype.storm.StormSubmitter; -import backtype.storm.generated.Grouping; -import backtype.storm.spout.RawScheme; -import backtype.storm.spout.SchemeAsMultiScheme; -import backtype.storm.topology.BoltDeclarer; -import backtype.storm.topology.TopologyBuilder; -import backtype.storm.tuple.Fields; - -import com.esotericsoftware.kryo.serializers.FieldSerializer; -import com.esotericsoftware.kryo.serializers.MapSerializer; - - - -import com.opensoc.alerts.TelemetryAlertsBolt; -import com.opensoc.alerts.adapters.HbaseWhiteAndBlacklistAdapter; -import com.opensoc.alerts.interfaces.AlertsAdapter; -import com.opensoc.enrichment.adapters.cif.CIFHbaseAdapter; -import com.opensoc.enrichment.adapters.geo.GeoMysqlAdapter; -import com.opensoc.enrichment.adapters.host.HostFromPropertiesFileAdapter; -import com.opensoc.enrichment.adapters.whois.WhoisHBaseAdapter; -import com.opensoc.enrichment.adapters.threat.ThreatHbaseAdapter; -import com.opensoc.enrichment.common.GenericEnrichmentBolt; -import com.opensoc.enrichment.interfaces.EnrichmentAdapter; -import com.opensoc.hbase.HBaseBolt; -import com.opensoc.hbase.HBaseStreamPartitioner; -import com.opensoc.hbase.TupleTableConfig; -import com.opensoc.helpers.topology.Cli; -import com.opensoc.helpers.topology.SettingsLoader; -import com.opensoc.index.interfaces.IndexAdapter; -import com.opensoc.indexing.TelemetryIndexingBolt; -import com.opensoc.json.serialization.JSONKryoSerializer; - -public abstract class TopologyRunner { - - protected Configuration config; - protected TopologyBuilder builder; - protected Config conf; - protected boolean local_mode = true; - protected boolean debug = true; - protected String config_path = null; - protected String default_config_path = "OpenSOC_Configs"; - protected boolean success = false; - protected Stack messageComponents = new Stack(); - protected Stack errorComponents = new Stack(); - protected Stack alertComponents = new Stack(); - protected Stack dataComponents = new Stack(); - protected Stack terminalComponents = new Stack(); - - public void initTopology(String args[], String subdir) - throws Exception { - Cli command_line = new Cli(args); - command_line.parse(); - - System.out.println("[OpenSOC] Starting topology deployment..."); - - debug = command_line.isDebug(); - System.out.println("[OpenSOC] Debug mode set to: " + debug); - - local_mode = command_line.isLocal_mode(); - System.out.println("[OpenSOC] Local mode set to: " + local_mode); - - if (command_line.getPath() != null) { - config_path = command_line.getPath(); - System.out - .println("[OpenSOC] Setting config path to external config path: " - + config_path); - } else { - config_path = default_config_path; - System.out - .println("[OpenSOC] Initializing from default internal config path: " - + config_path); - } - - String topology_conf_path = config_path + "/topologies/" + subdir - + "/topology.conf"; - - String environment_identifier_path = config_path - + "/topologies/environment_identifier.conf"; - String topology_identifier_path = config_path + "/topologies/" + subdir - + "/topology_identifier.conf"; - - System.out.println("[OpenSOC] Looking for environment identifier: " - + environment_identifier_path); - System.out.println("[OpenSOC] Looking for topology identifier: " - + topology_identifier_path); - System.out.println("[OpenSOC] Looking for topology config: " - + topology_conf_path); - - config = new PropertiesConfiguration(topology_conf_path); - - JSONObject environment_identifier = SettingsLoader - .loadEnvironmentIdnetifier(environment_identifier_path); - JSONObject topology_identifier = SettingsLoader - .loadTopologyIdnetifier(topology_identifier_path); - - String topology_name = SettingsLoader.generateTopologyName( - environment_identifier, topology_identifier); - - System.out.println("[OpenSOC] Initializing Topology: " + topology_name); - - builder = new TopologyBuilder(); - - conf = new Config(); - conf.registerSerialization(JSONObject.class, MapSerializer.class); - conf.setDebug(debug); - - System.out.println("[OpenSOC] Initializing Spout: " + topology_name); - - if (command_line.isGenerator_spout()) { - String component_name = config.getString("spout.test.name", - "DefaultTopologySpout"); - success = initializeTestingSpout(component_name); - messageComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "spout.test"); - } - - if (!command_line.isGenerator_spout()) { - String component_name = config.getString("spout.kafka.name", - "DefaultTopologyKafkaSpout"); - - success = initializeKafkaSpout(component_name); - messageComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "spout.kafka"); - } - - if (config.getBoolean("bolt.parser.enabled", true)) { - String component_name = config.getString("bolt.parser.name", - "DefaultTopologyParserBot"); - - success = initializeParsingBolt(topology_name, component_name); - messageComponents.add(component_name); - errorComponents.add(component_name); - - dataComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.parser"); - } - - if (config.getBoolean("bolt.enrichment.geo.enabled", false)) { - String component_name = config.getString( - "bolt.enrichment.geo.name", "DefaultGeoEnrichmentBolt"); - - success = initializeGeoEnrichment(topology_name, component_name); - messageComponents.add(component_name); - errorComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.enrichment.geo"); - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "mysql"); - } - - if (config.getBoolean("bolt.enrichment.host.enabled", false)) { - String component_name = config.getString( - "bolt.enrichment.host.name", "DefaultHostEnrichmentBolt"); - - success = initializeHostsEnrichment(topology_name, component_name, - "OpenSOC_Configs/etc/whitelists/known_hosts.conf"); - messageComponents.add(component_name); - errorComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.enrichment.host"); - } - - if (config.getBoolean("bolt.enrichment.whois.enabled", false)) { - String component_name = config.getString( - "bolt.enrichment.whois.name", "DefaultWhoisEnrichmentBolt"); - - success = initializeWhoisEnrichment(topology_name, component_name); - messageComponents.add(component_name); - errorComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.enrichment.whois"); - } - - if (config.getBoolean("bolt.enrichment.cif.enabled", false)) { - String component_name = config.getString( - "bolt.enrichment.cif.name", "DefaultCIFEnrichmentBolt"); - - success = initializeCIFEnrichment(topology_name, component_name); - messageComponents.add(component_name); - errorComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.enrichment.cif"); - } - - if (config.getBoolean("bolt.enrichment.threat.enabled", false)) { - String component_name = config.getString( - "bolt.enrichment.threat.name", "DefaultThreatEnrichmentBolt"); - - success = initializeThreatEnrichment(topology_name, component_name); - messageComponents.add(component_name); - errorComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.enrichment.threat"); - } - - if (config.getBoolean("bolt.alerts.enabled", false)) { - String component_name = config.getString("bolt.alerts.name", - "DefaultAlertsBolt"); - - success = initializeAlerts(topology_name, component_name, - config_path + "/topologies/" + subdir + "/alerts.xml", - environment_identifier, topology_identifier); - - messageComponents.add(component_name); - errorComponents.add(component_name); - alertComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.alerts"); - } - - if (config.getBoolean("bolt.alerts.indexing.enabled") && config.getBoolean("bolt.alerts.enabled")) { - - String component_name = config.getString( - "bolt.alerts.indexing.name", "DefaultAlertsBolt"); - - success = initializeAlertIndexing(component_name); - terminalComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.alerts.indexing"); - } - - if (config.getBoolean("bolt.kafka.enabled", false)) { - String component_name = config.getString("bolt.kafka.name", - "DefaultKafkaBolt"); - - success = initializeKafkaBolt(component_name); - terminalComponents.add(component_name); - - System.out.println("[OpenSOC] Component " + component_name - + " initialized"); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.kafka"); - } - - if (config.getBoolean("bolt.indexing.enabled", true)) { - String component_name = config.getString("bolt.indexing.name", - "DefaultIndexingBolt"); - - success = initializeIndexingBolt(component_name); - errorComponents.add(component_name); - terminalComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.indexing"); - } - - if (config.getBoolean("bolt.hdfs.enabled", false)) { - String component_name = config.getString("bolt.hdfs.name", - "DefaultHDFSBolt"); - - success = initializeHDFSBolt(topology_name, component_name); - terminalComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.hdfs"); - } - - if (config.getBoolean("bolt.error.indexing.enabled")) { - String component_name = config.getString( - "bolt.error.indexing.name", "DefaultErrorIndexingBolt"); - - success = initializeErrorIndexBolt(component_name); - terminalComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.error"); - } - - if (config.containsKey("bolt.hbase.enabled") - && config.getBoolean("bolt.hbase.enabled")) { - String component_name = config.getString("bolt.hbase.name", - "DefaultHbaseBolt"); - - String shuffleType = config.getString("bolt.hbase.shuffle.type", - "direct"); - success = initializeHbaseBolt(component_name, shuffleType); - terminalComponents.add(component_name); - - System.out.println("[OpenSOC] ------Component " + component_name - + " initialized with the following settings:"); - - SettingsLoader.printConfigOptions((PropertiesConfiguration) config, - "bolt.hbase"); - } - - System.out.println("[OpenSOC] Topology Summary: "); - System.out.println("[OpenSOC] Message Stream: " - + printComponentStream(messageComponents)); - System.out.println("[OpenSOC] Alerts Stream: " - + printComponentStream(alertComponents)); - System.out.println("[OpenSOC] Error Stream: " - + printComponentStream(errorComponents)); - System.out.println("[OpenSOC] Data Stream: " - + printComponentStream(dataComponents)); - System.out.println("[OpenSOC] Terminal Components: " - + printComponentStream(terminalComponents)); - - if (local_mode) { - conf.setNumWorkers(config.getInt("num.workers")); - conf.setMaxTaskParallelism(1); - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology(topology_name, conf, - builder.createTopology()); - } else { - - conf.setNumWorkers(config.getInt("num.workers")); - conf.setNumAckers(config.getInt("num.ackers")); - StormSubmitter.submitTopology(topology_name, conf, - builder.createTopology()); - } - - } - - private String printComponentStream(List messageComponents) { - StringBuilder print_string = new StringBuilder(); - - for (String component : messageComponents) { - print_string.append(component + " -> "); - } - - print_string.append("[TERMINAL COMPONENT]"); - - return print_string.toString(); - } - - public boolean initializeHbaseBolt(String name, String shuffleType) { - - try { - - String messageUpstreamComponent = dataComponents.get(dataComponents - .size()-1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - String tableName = config.getString("bolt.hbase.table.name") - .toString(); - TupleTableConfig hbaseBoltConfig = new TupleTableConfig(tableName, - config.getString("bolt.hbase.table.key.tuple.field.name") - .toString(), config.getString( - "bolt.hbase.table.timestamp.tuple.field.name") - .toString()); - - String allColumnFamiliesColumnQualifiers = config.getString( - "bolt.hbase.table.fields").toString(); - // This is expected in the form - // ":,,|:,|......." - String[] tokenizedColumnFamiliesWithColumnQualifiers = StringUtils - .split(allColumnFamiliesColumnQualifiers, "\\|"); - for (String tokenizedColumnFamilyWithColumnQualifiers : tokenizedColumnFamiliesWithColumnQualifiers) { - String[] cfCqTokens = StringUtils.split( - tokenizedColumnFamilyWithColumnQualifiers, ":"); - String columnFamily = cfCqTokens[0]; - String[] columnQualifiers = StringUtils.split(cfCqTokens[1], - ","); - for (String columnQualifier : columnQualifiers) { - hbaseBoltConfig.addColumn(columnFamily, columnQualifier); - } - - // hbaseBoltConfig.setDurability(Durability.valueOf(conf.get( - // "storm.topology.pcap.bolt.hbase.durability").toString())); - - hbaseBoltConfig.setBatch(Boolean.valueOf(config.getString( - "bolt.hbase.enable.batching").toString())); - - HBaseBolt hbase_bolt = new HBaseBolt(hbaseBoltConfig, - config.getString("kafka.zk.list"), - config.getString("kafka.zk.port")); - hbase_bolt.setAutoAck(true); - - BoltDeclarer declarer = builder.setBolt(name, hbase_bolt, - config.getInt("bolt.hbase.parallelism.hint")) - .setNumTasks(config.getInt("bolt.hbase.num.tasks")); - - if (Grouping._Fields.CUSTOM_OBJECT.toString().equalsIgnoreCase( - shuffleType)) { - declarer.customGrouping( - messageUpstreamComponent, - "pcap_data_stream", - new HBaseStreamPartitioner( - hbaseBoltConfig.getTableName(), - 0, - Integer.parseInt(conf - .get("bolt.hbase.partitioner.region.info.refresh.interval.mins") - .toString()))); - } else if (Grouping._Fields.DIRECT.toString().equalsIgnoreCase( - shuffleType)) { - declarer.fieldsGrouping(messageUpstreamComponent, - "pcap_data_stream", new Fields("pcap_id")); - } - - } - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - private boolean initializeErrorIndexBolt(String component_name) { - try { - - Class loaded_class = Class.forName(config.getString("bolt.error.indexing.adapter")); - IndexAdapter adapter = (IndexAdapter) loaded_class.newInstance(); - - String dateFormat = "yyyy.MM"; - if (config.containsKey("bolt.alerts.indexing.timestamp")) { - dateFormat = config.getString("bolt.alerts.indexing.timestamp"); - } - - TelemetryIndexingBolt indexing_bolt = new TelemetryIndexingBolt() - .withIndexIP(config.getString("es.ip")) - .withIndexPort(config.getInt("es.port")) - .withClusterName(config.getString("es.clustername")) - .withIndexName( - config.getString("bolt.error.indexing.indexname")) - .withDocumentName( - config.getString("bolt.error.indexing.documentname")) - .withIndexTimestamp(dateFormat) - .withBulk(config.getInt("bolt.error.indexing.bulk")) - .withIndexAdapter(adapter) - .withMetricConfiguration(config); - - BoltDeclarer declarer = builder - .setBolt( - component_name, - indexing_bolt, - config.getInt("bolt.error.indexing.parallelism.hint")) - .setNumTasks(config.getInt("bolt.error.indexing.num.tasks")); - - for (String component : errorComponents) - declarer.shuffleGrouping(component, "error"); - - return true; - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - private boolean initializeKafkaSpout(String name) { - try { - - BrokerHosts zk = new ZkHosts(config.getString("kafka.zk")); - String input_topic = config.getString("spout.kafka.topic"); - SpoutConfig kafkaConfig = new SpoutConfig(zk, input_topic, "", - input_topic); - kafkaConfig.scheme = new SchemeAsMultiScheme(new RawScheme()); - kafkaConfig.startOffsetTime = -1; - - builder.setSpout(name, new KafkaSpout(kafkaConfig), - config.getInt("spout.kafka.parallelism.hint")).setNumTasks( - config.getInt("spout.kafka.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - abstract boolean initializeParsingBolt(String topology_name, String name); - - abstract boolean initializeTestingSpout(String name); - - private boolean initializeGeoEnrichment(String topology_name, String name) { - - try { - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - - String[] keys_from_settings = config.getStringArray("bolt.enrichment.geo.fields"); - List geo_keys = new ArrayList(Arrays.asList(keys_from_settings)); - - GeoMysqlAdapter geo_adapter = new GeoMysqlAdapter( - config.getString("mysql.ip"), config.getInt("mysql.port"), - config.getString("mysql.username"), - config.getString("mysql.password"), - config.getString("bolt.enrichment.geo.adapter.table")); - - GenericEnrichmentBolt geo_enrichment = new GenericEnrichmentBolt() - .withEnrichmentTag( - config.getString("bolt.enrichment.geo.enrichment_tag")) - .withOutputFieldName(topology_name) - .withAdapter(geo_adapter) - .withMaxTimeRetain( - config.getInt("bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES")) - .withMaxCacheSize( - config.getInt("bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM")) - .withKeys(geo_keys).withMetricConfiguration(config); - - builder.setBolt(name, geo_enrichment, - config.getInt("bolt.enrichment.geo.parallelism.hint")) - .fieldsGrouping(messageUpstreamComponent, "message", - new Fields("key")) - .setNumTasks(config.getInt("bolt.enrichment.geo.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - private boolean initializeHostsEnrichment(String topology_name, - String name, String hosts_path) { - - try { - - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - List hosts_keys = new ArrayList(); - hosts_keys.add(config.getString("source.ip")); - hosts_keys.add(config.getString("dest.ip")); - - Map known_hosts = SettingsLoader - .loadKnownHosts(hosts_path); - - HostFromPropertiesFileAdapter host_adapter = new HostFromPropertiesFileAdapter( - known_hosts); - - GenericEnrichmentBolt host_enrichment = new GenericEnrichmentBolt() - .withEnrichmentTag( - config.getString("bolt.enrichment.host.enrichment_tag")) - .withAdapter(host_adapter) - .withMaxTimeRetain( - config.getInt("bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES")) - .withMaxCacheSize( - config.getInt("bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM")) - .withOutputFieldName(topology_name).withKeys(hosts_keys) - .withMetricConfiguration(config); - - builder.setBolt(name, host_enrichment, - config.getInt("bolt.enrichment.host.parallelism.hint")) - .fieldsGrouping(messageUpstreamComponent, "message", - new Fields("key")) - .setNumTasks( - config.getInt("bolt.enrichment.host.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - @SuppressWarnings("rawtypes") - private boolean initializeAlerts(String topology_name, String name, - String alerts_path, JSONObject environment_identifier, - JSONObject topology_identifier) { - try { - - Class loaded_class = Class.forName(config.getString("bolt.alerts.adapter")); - Constructor constructor = loaded_class.getConstructor(new Class[] { Map.class}); - - Map settings = SettingsLoader.getConfigOptions((PropertiesConfiguration)config, config.getString("bolt.alerts.adapter") + "."); - - System.out.println("Adapter Settings: "); - SettingsLoader.printOptionalSettings(settings); - - AlertsAdapter alerts_adapter = (AlertsAdapter) constructor.newInstance(settings); - - - - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - JSONObject alerts_identifier = SettingsLoader - .generateAlertsIdentifier(environment_identifier, - topology_identifier); - - - - TelemetryAlertsBolt alerts_bolt = new TelemetryAlertsBolt() - .withIdentifier(alerts_identifier).withMaxCacheSize(1000) - .withMaxTimeRetain(3600).withAlertsAdapter(alerts_adapter) - .withOutputFieldName("message") - .withMetricConfiguration(config); - - builder.setBolt(name, alerts_bolt, - config.getInt("bolt.alerts.parallelism.hint")) - .fieldsGrouping(messageUpstreamComponent, "message", - new Fields("key")) - .setNumTasks(config.getInt("bolt.alerts.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - private boolean initializeAlertIndexing(String name) { - - try{ - String messageUpstreamComponent = alertComponents.get(alertComponents - .size() - 1); - - System.out.println("[OpenSOC] ------" + name + " is initializing from " - + messageUpstreamComponent); - - Class loaded_class = Class.forName(config.getString("bolt.alerts.indexing.adapter")); - IndexAdapter adapter = (IndexAdapter) loaded_class.newInstance(); - - String dateFormat = "yyyy.MM.dd"; - if (config.containsKey("bolt.alerts.indexing.timestamp")) { - dateFormat = config.getString("bolt.alerts.indexing.timestamp"); - } - TelemetryIndexingBolt indexing_bolt = new TelemetryIndexingBolt() - .withIndexIP(config.getString("es.ip")) - .withIndexPort(config.getInt("es.port")) - .withClusterName(config.getString("es.clustername")) - .withIndexName( - config.getString("bolt.alerts.indexing.indexname")) - .withDocumentName( - config.getString("bolt.alerts.indexing.documentname")) - .withIndexTimestamp(dateFormat) - .withBulk(config.getInt("bolt.alerts.indexing.bulk")) - .withIndexAdapter(adapter) - .withMetricConfiguration(config); - - String alerts_name = config.getString("bolt.alerts.indexing.name"); - builder.setBolt(alerts_name, indexing_bolt, - config.getInt("bolt.indexing.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent, "alert") - .setNumTasks(config.getInt("bolt.indexing.num.tasks")); - } - catch(Exception e) - { - e.printStackTrace(); - return false; - } - - return true; - } - - private boolean initializeKafkaBolt(String name) { - try { - - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - Map kafka_broker_properties = new HashMap(); - kafka_broker_properties.put("zk.connect", - config.getString("kafka.zk")); - kafka_broker_properties.put("metadata.broker.list", - config.getString("kafka.br")); - - kafka_broker_properties.put("serializer.class", - "com.opensoc.json.serialization.JSONKafkaSerializer"); - - kafka_broker_properties.put("key.serializer.class", - "kafka.serializer.StringEncoder"); - - String output_topic = config.getString("bolt.kafka.topic"); - - conf.put("kafka.broker.properties", kafka_broker_properties); - conf.put("topic", output_topic); - - builder.setBolt(name, new KafkaBolt(), - config.getInt("bolt.kafka.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent, "message") - .setNumTasks(config.getInt("bolt.kafka.num.tasks")); - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - return true; - } - - private boolean initializeWhoisEnrichment(String topology_name, String name) { - try { - - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - String[] keys_from_settings = config.getString("bolt.enrichment.whois.fields").split(","); - List whois_keys = new ArrayList(Arrays.asList(keys_from_settings)); - - EnrichmentAdapter whois_adapter = new WhoisHBaseAdapter( - config.getString("bolt.enrichment.whois.hbase.table.name"), - config.getString("kafka.zk.list"), - config.getString("kafka.zk.port")); - - GenericEnrichmentBolt whois_enrichment = new GenericEnrichmentBolt() - .withEnrichmentTag( - config.getString("bolt.enrichment.whois.enrichment_tag")) - .withOutputFieldName(topology_name) - .withAdapter(whois_adapter) - .withMaxTimeRetain( - config.getInt("bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES")) - .withMaxCacheSize( - config.getInt("bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM")) - .withKeys(whois_keys).withMetricConfiguration(config); - - builder.setBolt(name, whois_enrichment, - config.getInt("bolt.enrichment.whois.parallelism.hint")) - .fieldsGrouping(messageUpstreamComponent, "message", - new Fields("key")) - .setNumTasks( - config.getInt("bolt.enrichment.whois.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - private boolean initializeIndexingBolt(String name) { - try { - - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - Class loaded_class = Class.forName(config.getString("bolt.indexing.adapter")); - IndexAdapter adapter = (IndexAdapter) loaded_class.newInstance(); - - Map settings = SettingsLoader.getConfigOptions((PropertiesConfiguration)config, "optional.settings.bolt.index.search."); - - if(settings != null && settings.size() > 0) - { - adapter.setOptionalSettings(settings); - System.out.println("[OpenSOC] Index Bolt picket up optional settings:"); - SettingsLoader.printOptionalSettings(settings); - } - - // dateFormat defaults to hourly if not specified - String dateFormat = "yyyy.MM.dd.hh"; - if (config.containsKey("bolt.indexing.timestamp")) { - dateFormat = config.getString("bolt.indexing.timestamp"); - } - TelemetryIndexingBolt indexing_bolt = new TelemetryIndexingBolt() - .withIndexIP(config.getString("es.ip")) - .withIndexPort(config.getInt("es.port")) - .withClusterName(config.getString("es.clustername")) - .withIndexName(config.getString("bolt.indexing.indexname")) - .withIndexTimestamp(dateFormat) - .withDocumentName( - config.getString("bolt.indexing.documentname")) - .withBulk(config.getInt("bolt.indexing.bulk")) - .withIndexAdapter(adapter) - .withMetricConfiguration(config); - - builder.setBolt(name, indexing_bolt, - config.getInt("bolt.indexing.parallelism.hint")) - .fieldsGrouping(messageUpstreamComponent, "message", - new Fields("key")) - .setNumTasks(config.getInt("bolt.indexing.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - - private boolean initializeThreatEnrichment(String topology_name, String name) { - try { - - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - String[] fields = config.getStringArray("bolt.enrichment.threat.fields"); - List threat_keys = new ArrayList(Arrays.asList(fields)); - - GenericEnrichmentBolt threat_enrichment = new GenericEnrichmentBolt() - .withEnrichmentTag( - config.getString("bolt.enrichment.threat.enrichment_tag")) - .withAdapter( - new ThreatHbaseAdapter(config - .getString("kafka.zk.list"), config - .getString("kafka.zk.port"), config - .getString("bolt.enrichment.threat.tablename"))) - .withOutputFieldName(topology_name) - .withEnrichmentTag(config.getString("bolt.enrichment.threat.enrichment_tag")) - .withKeys(threat_keys) - .withMaxTimeRetain( - config.getInt("bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES")) - .withMaxCacheSize( - config.getInt("bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM")) - .withMetricConfiguration(config); - - builder.setBolt(name, threat_enrichment, - config.getInt("bolt.enrichment.threat.parallelism.hint")) - .fieldsGrouping(messageUpstreamComponent, "message", - new Fields("key")) - .setNumTasks(config.getInt("bolt.enrichment.threat.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - private boolean initializeCIFEnrichment(String topology_name, String name) { - try { - - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - List cif_keys = new ArrayList(); - - String[] ipFields = config.getStringArray("bolt.enrichment.cif.fields.ip"); - cif_keys.addAll(Arrays.asList(ipFields)); - - String[] hostFields = config.getStringArray("bolt.enrichment.cif.fields.host"); - cif_keys.addAll(Arrays.asList(hostFields)); - - String[] emailFields = config.getStringArray("bolt.enrichment.cif.fields.email"); - cif_keys.addAll(Arrays.asList(emailFields)); - - GenericEnrichmentBolt cif_enrichment = new GenericEnrichmentBolt() - .withEnrichmentTag( - config.getString("bolt.enrichment.cif.enrichment_tag")) - .withAdapter( - new CIFHbaseAdapter(config - .getString("kafka.zk.list"), config - .getString("kafka.zk.port"), config - .getString("bolt.enrichment.cif.tablename"))) - .withOutputFieldName(topology_name) - .withKeys(cif_keys) - .withMaxTimeRetain( - config.getInt("bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES")) - .withMaxCacheSize( - config.getInt("bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM")) - .withMetricConfiguration(config); - - builder.setBolt(name, cif_enrichment, - config.getInt("bolt.enrichment.cif.parallelism.hint")) - .fieldsGrouping(messageUpstreamComponent, "message", - new Fields("key")) - .setNumTasks(config.getInt("bolt.enrichment.cif.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - - private boolean initializeHDFSBolt(String topology_name, String name) { - try { - - String messageUpstreamComponent = messageComponents - .get(messageComponents.size() - 1); - - System.out.println("[OpenSOC] ------" + name - + " is initializing from " + messageUpstreamComponent); - - RecordFormat format = new DelimitedRecordFormat() - .withFieldDelimiter( - config.getString("bolt.hdfs.field.delimiter") - .toString()).withFields( - new Fields("message")); - - // sync the file system after every x number of tuples - SyncPolicy syncPolicy = new CountSyncPolicy(Integer.valueOf(config - .getString("bolt.hdfs.batch.size").toString())); - - // rotate files when they reach certain size - FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy( - Float.valueOf(config.getString( - "bolt.hdfs.file.rotation.size.in.mb").toString()), - Units.MB); - - FileNameFormat fileNameFormat = new DefaultFileNameFormat() - .withPath(config.getString("bolt.hdfs.wip.file.path") - .toString()); - - // Post rotate action - MoveFileAction moveFileAction = (new MoveFileAction()) - .toDestination(config.getString( - "bolt.hdfs.finished.file.path").toString()); - - HdfsBolt hdfsBolt = new HdfsBolt() - .withFsUrl( - config.getString("bolt.hdfs.file.system.url") - .toString()) - .withFileNameFormat(fileNameFormat) - .withRecordFormat(format) - .withRotationPolicy(rotationPolicy) - .withSyncPolicy(syncPolicy) - .addRotationAction(moveFileAction); - if (config.getString("bolt.hdfs.compression.codec.class") != null) { - hdfsBolt.withCompressionCodec(config.getString( - "bolt.hdfs.compression.codec.class").toString()); - } - - builder.setBolt(name, hdfsBolt, - config.getInt("bolt.hdfs.parallelism.hint")) - .shuffleGrouping(messageUpstreamComponent, "message") - .setNumTasks(config.getInt("bolt.hdfs.num.tasks")); - - } catch (Exception e) { - e.printStackTrace(); - System.exit(0); - } - - return true; - } - -} From bc1b08b1bda88ebe0872aee69de4d13799bb2d56 Mon Sep 17 00:00:00 2001 From: Sean Schulte Date: Wed, 20 Jan 2016 12:09:05 -0600 Subject: [PATCH 7/7] Added a FalconHose topology. This includes a message parser, as well as a new alert adapter that knows that everything from the FalconHose should be considered an alert. There are Flux YAML file templates for both local and remote running. --- .../adapters/FalconHoseAlertAdapter.java | 87 ++++++ .../adapters/FalconHoseAlertAdapterTest.java | 93 ++++++ .../parsers/BasicFalconHoseParser.java | 88 ++++++ .../test/BasicFalconHoseParserTest.java | 76 +++++ .../topologies/falconhose/local.yaml | 260 +++++++++++++++++ .../topologies/falconhose/remote.yaml | 272 ++++++++++++++++++ .../SampleInput/FalconHoseExampleOutput | 4 + 7 files changed, 880 insertions(+) create mode 100644 metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/FalconHoseAlertAdapter.java create mode 100644 metron-streaming/Metron-Alerts/src/test/java/org/apache/metron/alerts/adapters/FalconHoseAlertAdapterTest.java create mode 100644 metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicFalconHoseParser.java create mode 100644 metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/test/BasicFalconHoseParserTest.java create mode 100644 metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/falconhose/local.yaml create mode 100644 metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/falconhose/remote.yaml create mode 100644 metron-streaming/Metron-Topologies/src/main/resources/SampleInput/FalconHoseExampleOutput diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/FalconHoseAlertAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/FalconHoseAlertAdapter.java new file mode 100644 index 0000000000..8a7da8f42e --- /dev/null +++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/FalconHoseAlertAdapter.java @@ -0,0 +1,87 @@ +package org.apache.metron.alerts.adapters; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.json.simple.JSONObject; +import org.apache.log4j.Logger; +import org.apache.metron.alerts.interfaces.AlertsAdapter; + +/** + * All messages from FalconHose are considered alerts, so this adapter + * simply constructs a normalized alert object for every message that comes + * in. + */ +public class FalconHoseAlertAdapter implements AlertsAdapter, Serializable { + + protected static final Logger LOG = Logger + .getLogger(FalconHoseAlertAdapter.class); + + public FalconHoseAlertAdapter() { + } + + public FalconHoseAlertAdapter(Map config) { + } + + @Override + public boolean initialize() { + return true; + } + + @Override + public boolean refresh() throws Exception { + return false; + } + + @Override + public Map alert(JSONObject rawMessage) { + Map alerts = new HashMap(); + + if (!rawMessage.containsKey("message")) { + return alerts; + } + + JSONObject content = (JSONObject)rawMessage.get("message"); + + String host = "unknown"; + if (content.containsKey("ip_dst_addr")) { + host = content.get("ip_dst_addr").toString(); + } + + String description = ""; + if (content.containsKey("original_string")) { + description = content.get("original_string").toString(); + } + + String alertId = generateAlertId(); + + JSONObject alert = new JSONObject(); + + alert.put("alert_id", alertId); + alert.put("designated_host", host); + alert.put("description", description); + + if (content.containsKey("SeverityName")) { + alert.put("priority", content.get("SeverityName").toString()); + } else { + alert.put("priority", "MED"); + } + + alerts.put(alertId, alert); + + return alerts; + } + + @Override + public boolean containsAlertId(String alert) { + return false; + } + + protected String generateAlertId() { + String new_UUID = System.currentTimeMillis() + "-" + UUID.randomUUID(); + return new_UUID; + + } +} diff --git a/metron-streaming/Metron-Alerts/src/test/java/org/apache/metron/alerts/adapters/FalconHoseAlertAdapterTest.java b/metron-streaming/Metron-Alerts/src/test/java/org/apache/metron/alerts/adapters/FalconHoseAlertAdapterTest.java new file mode 100644 index 0000000000..8eee60b61c --- /dev/null +++ b/metron-streaming/Metron-Alerts/src/test/java/org/apache/metron/alerts/adapters/FalconHoseAlertAdapterTest.java @@ -0,0 +1,93 @@ +package org.apache.metron.alerts.adapters; + +import java.util.Map; + +import org.json.simple.JSONObject; + +import org.apache.metron.test.AbstractConfigTest; +import org.apache.metron.alerts.adapters.FalconHoseAlertAdapter; + + /** + *
    + *
  • Title: FalconHoseAlertAdapterTest
    • + *
  • Description: Tests for FalconHoseAlertAdapter
    • + *
  • Created: January 20, 2016
    • + *
+ */ +public class FalconHoseAlertAdapterTest extends AbstractConfigTest { + + public FalconHoseAlertAdapterTest(String name) { + super(name); + } + + public void testInitializeAdapter() { + FalconHoseAlertAdapter adapter = new FalconHoseAlertAdapter(); + boolean initialized = adapter.initialize(); + assertTrue(initialized); + } + + public void testRefresh() throws Exception { + FalconHoseAlertAdapter adapter = new FalconHoseAlertAdapter(); + boolean refreshed = adapter.refresh(); + assertFalse(refreshed); + } + + public void testContainsAlertId(){ + FalconHoseAlertAdapter adapter = new FalconHoseAlertAdapter(); + boolean containsAlert = adapter.containsAlertId("test"); + assertFalse(containsAlert); + } + + public void testAlertNoMessage() { + FalconHoseAlertAdapter adapter = new FalconHoseAlertAdapter(); + + JSONObject message = new JSONObject(); + + Map alerts = adapter.alert(message); + + assertEquals(0, alerts.size()); + } + + public void testAlertEmptyMessage() { + FalconHoseAlertAdapter adapter = new FalconHoseAlertAdapter(); + + JSONObject internalMessage = new JSONObject(); + JSONObject message = new JSONObject(); + message.put("message", internalMessage); + + Map alerts = adapter.alert(message); + + assertEquals(1, alerts.size()); + + String alertId = alerts.keySet().iterator().next(); + JSONObject alert = alerts.get(alertId); + assertEquals(alertId, alert.get("alert_id")); + assertEquals("unknown", alert.get("designated_host")); + assertEquals("", alert.get("description")); + assertEquals("MED", alert.get("priority")); + } + + public void testAlert() { + FalconHoseAlertAdapter adapter = new FalconHoseAlertAdapter(); + + JSONObject internalMessage = new JSONObject(); + internalMessage.put("ip_dst_addr", "192.168.0.50"); + internalMessage.put("original_string", "this is original"); + internalMessage.put("SeverityName", "High"); + JSONObject message = new JSONObject(); + message.put("message", internalMessage); + + Map alerts = adapter.alert(message); + + assertEquals(1, alerts.size()); + + String alertId = alerts.keySet().iterator().next(); + JSONObject alert = alerts.get(alertId); + assertEquals(alertId, alert.get("alert_id")); + assertEquals("192.168.0.50", alert.get("designated_host")); + assertEquals("this is original", alert.get("description")); + assertEquals("High", alert.get("priority")); + } + +} + diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicFalconHoseParser.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicFalconHoseParser.java new file mode 100644 index 0000000000..5d1cc65bca --- /dev/null +++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicFalconHoseParser.java @@ -0,0 +1,88 @@ +package org.apache.metron.parsing.parsers; + +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.metron.tldextractor.BasicTldExtractor; + +@SuppressWarnings("serial") +public class BasicFalconHoseParser extends AbstractParser { + + protected static final Logger _LOG = LoggerFactory.getLogger(BasicFalconHoseParser.class); + private JSONCleaner cleaner = new JSONCleaner(); + + @SuppressWarnings("unchecked") + public JSONObject parse(byte[] msg) { + _LOG.trace("[OpenSOC] Starting to parse incoming message"); + + String rawMessage = null; + + try { + + rawMessage = new String(msg, "UTF-8"); + _LOG.trace("[OpenSOC] Received message: " + rawMessage); + + JSONObject cleanedMessage = cleaner.Clean(rawMessage); + _LOG.debug("[OpenSOC] Cleaned message: " + rawMessage); + + if (cleanedMessage == null || cleanedMessage.isEmpty()) { + throw new Exception("Unable to clean message: " + rawMessage); + } + + JSONObject payload = (JSONObject)cleanedMessage.get("event"); + + if (payload == null) { + throw new Exception("Unable to retrieve payload for message: " + + rawMessage); + } + + String originalString = ""; + for (Object k : payload.keySet()) { + originalString += " " + k.toString() + ":" + payload.get(k).toString(); + } + payload.put("original_string", originalString); + + if (payload.containsKey("LoginTime")) { + Long ts = Long.parseLong(payload.remove("LoginTime").toString()); + payload.put("timestamp", ts * 1000); + _LOG.trace("[OpenSOC] Added ts to: " + payload); + } else if (payload.containsKey("ProcessStartTime")) { + Long ts = Long.parseLong(payload.remove("ProcessStartTime").toString()); + payload.put("timestamp", ts); + _LOG.trace("[OpenSOC] Added ts to: " + payload); + } else { + payload.put("timestamp", System.currentTimeMillis()); + } + + if (payload.containsKey("UserIp")) { + String ip = payload.remove("UserIp").toString(); + payload.put("ip_src_addr", ip); + payload.put("ip_dst_addr", ip); + payload.put("ip_src_port", 0); + payload.put("ip_dst_port", 0); + } else if (payload.containsKey("ComputerName")) { + String name = payload.remove("ComputerName").toString(); + payload.put("ip_src_addr", name); + payload.put("ip_dst_addr", name); + payload.put("ip_src_port", 0); + payload.put("ip_dst_port", 0); + } + + _LOG.trace("[OpenSOC] Inner message: " + payload); + + payload.put("protocol", "http"); + _LOG.debug("[OpenSOC] Returning parsed message: " + payload); + + return payload; + } catch (Exception e) { + _LOG.error("Unable to Parse Message: " + rawMessage); + _LOG.error(e.getMessage(), e); + return null; + } + + } + + +} diff --git a/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/test/BasicFalconHoseParserTest.java b/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/test/BasicFalconHoseParserTest.java new file mode 100644 index 0000000000..0f6da129de --- /dev/null +++ b/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/test/BasicFalconHoseParserTest.java @@ -0,0 +1,76 @@ +package org.apache.metron.parsing.test; + +import java.util.Map; + +import junit.framework.TestCase; + +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import org.apache.metron.parsing.parsers.BasicFalconHoseParser; + +public class BasicFalconHoseParserTest extends TestCase { + + private BasicFalconHoseParser falconHoseParser = null; + private JSONParser jsonParser = null; + + public BasicFalconHoseParserTest() throws Exception { + falconHoseParser = new BasicFalconHoseParser(); + jsonParser = new JSONParser(); + } + + public void testLoginAuditEvent() throws ParseException { + String rawMessage = "{\"metadata\":{\"offset\":3302,\"eventType\":\"LoginAuditEvent\"},\"event\":{\"LoginTime\":1444160709766,\"UserId\":\"tyler.baker@customer.rackspace.com\",\"UserIp\":\"50.56.228.73\",\"OperationName\":\"UserAuthenticate\",\"ServiceName\":\"TokenApi\",\"Success\":true}}"; + + Map rawMessageMap = (Map) jsonParser.parse(rawMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get("event"); + + JSONObject fhJson = falconHoseParser.parse(rawMessage.getBytes()); + + assertEquals(Long.parseLong(fhJson.get("timestamp").toString()), Long.parseLong(rawJson.get("LoginTime").toString()) * 1000); + assertEquals(fhJson.get("ip_src_addr").toString(), rawJson.get("UserIp").toString()); + assertEquals(fhJson.get("ip_dst_addr").toString(), rawJson.get("UserIp").toString()); + assertEquals(fhJson.get("ip_src_port").toString(), "0"); + assertEquals(fhJson.get("ip_dst_port").toString(), "0"); + assertEquals(fhJson.get("protocol").toString(), "http"); + assertTrue(fhJson.containsKey("original_string")); + assertTrue(fhJson.containsKey("timestamp")); + } + + public void testDetectionSummaryEvent() throws ParseException { + String rawMessage = "{\"metadata\":{\"offset\":3304,\"eventType\":\"DetectionSummaryEvent\"},\"event\":{\"ProcessStartTime\":1444168443,\"ProcessEndTime\":0,\"ProcessId\":288437472047,\"ParentProcessId\":288435542004,\"ComputerName\":\"619027-DAPPP083\",\"UserName\":\"mxaon_admin\",\"DetectName\":\"Suspicious Activity\",\"DetectDescription\":\"An administrative/reconnaissance tool was spawned under an IIS worker process\",\"Severity\":2,\"SeverityName\":\"Low\",\"FileName\":\"regsvr32.exe\",\"FilePath\":\"\\Device\\HarddiskVolume1\\Windows\\SysWOW64\",\"CommandLine\":\"regsvr32.exe /u /s C:\\Windows\\system32\\dxtmsft.dll\",\"SHA256String\":\"890c1734ed1ef6b2422a9b21d6205cf91e014add8a7f41aa5a294fcf60631a7b\",\"MD5String\":\"432be6cf7311062633459eef6b242fb5\",\"SHA1String\":\"N/A\",\"MachineDomain\":\"619027-DAPPP083\",\"FalconHostLink\":\"https://falcon.crowdstrike.com/detects/-2623836595666801992\",\"SensorId\":\"97264ff9a8b548749f41871e09c6856e\"}}"; + + Map rawMessageMap = (Map) jsonParser.parse(rawMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get("event"); + + JSONObject fhJson = falconHoseParser.parse(rawMessage.getBytes()); + + assertEquals(Long.parseLong(fhJson.get("timestamp").toString()), Long.parseLong(rawJson.get("ProcessStartTime").toString())); + assertEquals(fhJson.get("ip_src_addr").toString(), rawJson.get("ComputerName").toString()); + assertEquals(fhJson.get("ip_dst_addr").toString(), rawJson.get("ComputerName").toString()); + assertEquals(fhJson.get("ip_src_port").toString(), "0"); + assertEquals(fhJson.get("ip_dst_port").toString(), "0"); + assertEquals(fhJson.get("protocol").toString(), "http"); + assertTrue(fhJson.containsKey("original_string")); + assertTrue(fhJson.containsKey("timestamp")); + } + + public void testUserActivityAuditEvent() throws ParseException { + String rawMessage = "{\"metadata\":{\"offset\":3326,\"eventType\":\"UserActivityAuditEvent\"},\"event\":{\"UserId\":\"jason.blagg@customer.rackspace.com\",\"UserIp\":\"50.56.228.68\",\"OperationName\":\"UpdateDetectState\",\"ServiceName\":\"Detects\",\"Success\":true,\"AuditKeyValues\":[{\"Key\":\"detects\",\"ValueString\":\"6574431533307329744\"},{\"Key\":\"new_state\",\"ValueString\":\"in_progress\"}]}}"; + + Map rawMessageMap = (Map) jsonParser.parse(rawMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get("event"); + + JSONObject fhJson = falconHoseParser.parse(rawMessage.getBytes()); + + assertEquals(fhJson.get("ip_src_addr").toString(), rawJson.get("UserIp").toString()); + assertEquals(fhJson.get("ip_dst_addr").toString(), rawJson.get("UserIp").toString()); + assertEquals(fhJson.get("ip_src_port").toString(), "0"); + assertEquals(fhJson.get("ip_dst_port").toString(), "0"); + assertEquals(fhJson.get("protocol").toString(), "http"); + assertTrue(fhJson.containsKey("original_string")); + assertTrue(fhJson.containsKey("timestamp")); + } +} diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/falconhose/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/falconhose/local.yaml new file mode 100644 index 0000000000..4254bbfa37 --- /dev/null +++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/falconhose/local.yaml @@ -0,0 +1,260 @@ +name: "falconhose-local" +config: + topology.workers: 1 + +components: + - id: "falconHoseParser" + className: "org.apache.metron.parsing.parsers.BasicFalconHoseParser" + - id: "genericMessageFilter" + className: "org.apache.metron.filters.GenericMessageFilter" + - id: "indexAdapter" + className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsAdapter" + className: "org.apache.metron.alerts.adapters.FalconHoseAlertAdapter" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "falconhose"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "org.apache.metron.metrics.reporter.graphite" + - "${org.apache.metron.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.reporter.console" + - "${org.apache.metron.metrics.reporter.console}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.reporter.jmx" + - "${org.apache.metron.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.graphite.address" + - "${org.apache.metron.metrics.graphite.address}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.graphite.port" + - "${org.apache.metron.metrics.graphite.port}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryParserBolt.acks" + - "${org.apache.metron.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryParserBolt.emits" + - "${org.apache.metron.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryParserBolt.fails" + - "${org.apache.metron.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.GenericEnrichmentBolt.acks" + - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.GenericEnrichmentBolt.emits" + - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.GenericEnrichmentBolt.fails" + - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryIndexingBolt.acks" + - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryIndexingBolt.emits" + - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryIndexingBolt.fails" + - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}" + +spouts: + - id: "testingSpout" + className: "org.apache.metron.test.spouts.GenericInternalTestSpout" + parallelism: 1 + configMethods: + - name: "withFilename" + args: + - "SampleInput/FalconHoseExampleOutput" + - name: "withRepeating" + args: + - true + +bolts: + - id: "parserBolt" + className: "org.apache.metron.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "falconHoseParser" + - name: "withOutputFieldName" + args: + - "falconhose" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "org.apache.metron.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "falconhose_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "falconhose_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "org.apache.metron.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "org.apache.metron.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "falconhose_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "org.apache.metron.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "falconhose_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "testingSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> indexing" + from: "parserBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "parser -> alerts" + from: "parserBolt" + to: "alertsBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "alert" + type: SHUFFLE + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/falconhose/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/falconhose/remote.yaml new file mode 100644 index 0000000000..7e95118bde --- /dev/null +++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/falconhose/remote.yaml @@ -0,0 +1,272 @@ +name: "falconhose" +config: + topology.workers: 1 + +components: + - id: "falconHoseParser" + className: "org.apache.metron.parsing.parsers.BasicFalconHoseParser" + - id: "genericMessageFilter" + className: "org.apache.metron.filters.GenericMessageFilter" + - id: "indexAdapter" + className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter" + - id: "alertsAdapter" + className: "org.apache.metron.alerts.adapters.FalconHoseAlertAdapter" + - id: "alertsIdentifier" + className: "org.json.simple.JSONObject" + configMethods: + - name: "put" + args: ["environment", "local"] + - name: "put" + args: ["topology", "falconhose"] + - id: "metricConfig" + className: "org.apache.commons.configuration.BaseConfiguration" + configMethods: + - name: "setProperty" + args: + - "org.apache.metron.metrics.reporter.graphite" + - "${org.apache.metron.metrics.reporter.graphite}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.reporter.console" + - "${org.apache.metron.metrics.reporter.console}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.reporter.jmx" + - "${org.apache.metron.metrics.reporter.jmx}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.graphite.address" + - "${org.apache.metron.metrics.graphite.address}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.graphite.port" + - "${org.apache.metron.metrics.graphite.port}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryParserBolt.acks" + - "${org.apache.metron.metrics.TelemetryParserBolt.acks}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryParserBolt.emits" + - "${org.apache.metron.metrics.TelemetryParserBolt.emits}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryParserBolt.fails" + - "${org.apache.metron.metrics.TelemetryParserBolt.fails}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.GenericEnrichmentBolt.acks" + - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.GenericEnrichmentBolt.emits" + - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.GenericEnrichmentBolt.fails" + - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryIndexingBolt.acks" + - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryIndexingBolt.emits" + - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}" + - name: "setProperty" + args: + - "org.apache.metron.metrics.TelemetryIndexingBolt.fails" + - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}" + - id: "zkHosts" + className: "storm.kafka.ZkHosts" + constructorArgs: + - "${kafka.zk}" + - id: "kafkaConfig" + className: "storm.kafka.SpoutConfig" + constructorArgs: + # zookeeper hosts + - ref: "zkHosts" + # topic name + - "${spout.kafka.topic}" + # zk root + - "" + # id + - "${spout.kafka.topic}" + properties: + - name: "startOffsetTime" + value: -1 + +spouts: + - id: "kafkaSpout" + className: "storm.kafka.KafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + +bolts: + - id: "parserBolt" + className: "org.apache.metron.parsing.TelemetryParserBolt" + configMethods: + - name: "withMessageParser" + args: + - ref: "falconHoseParser" + - name: "withOutputFieldName" + args: + - "falconhose" + - name: "withMessageFilter" + args: + - ref: "genericMessageFilter" + - name: "withMetricConfig" + args: + - ref: "metricConfig" + - id: "indexingBolt" + className: "org.apache.metron.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "falconhose_index" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.dd.hh" + - name: "withDocumentName" + args: + - "falconhose_doc" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsBolt" + className: "org.apache.metron.alerts.TelemetryAlertsBolt" + configMethods: + - name: "withIdentifier" + args: + - ref: "alertsIdentifier" + - name: "withMaxCacheSize" + args: [1000] + - name: "withMaxTimeRetain" + args: [3600] + - name: "withAlertsAdapter" + args: + - ref: "alertsAdapter" + - name: "withOutputFieldName" + args: ["message"] + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "alertsIndexingBolt" + className: "org.apache.metron.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "alert" + - name: "withIndexTimestamp" + args: + - "yyyy.MM.ww" + - name: "withDocumentName" + args: + - "falconhose_alert" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + - id: "errorIndexingBolt" + className: "org.apache.metron.indexing.TelemetryIndexingBolt" + configMethods: + - name: "withIndexIP" + args: + - "${es.ip}" + - name: "withIndexPort" + args: + - ${es.port} + - name: "withClusterName" + args: + - "${es.clustername}" + - name: "withIndexName" + args: + - "error" + - name: "withIndexTimestamp" + args: + - "yyyy.MM" + - name: "withDocumentName" + args: + - "falconhose_error" + - name: "withBulk" + args: + - 1 + - name: "withIndexAdapter" + args: + - ref: "indexAdapter" + - name: "withMetricConfiguration" + args: + - ref: "metricConfig" + +streams: + - name: "spout -> parser" + from: "kafkaSpout" + to: "parserBolt" + grouping: + type: SHUFFLE + - name: "parser -> indexing" + from: "parserBolt" + to: "indexingBolt" + grouping: + streamId: "message" + type: SHUFFLE + - name: "parser -> alerts" + from: "parserBolt" + to: "alertsBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + - name: "alerts -> alertsIndexing" + from: "alertsBolt" + to: "alertsIndexingBolt" + grouping: + streamId: "alert" + type: SHUFFLE + - name: "parser -> errors" + from: "parserBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "indexing -> errors" + from: "indexingBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE + - name: "alerts -> errors" + from: "alertsBolt" + to: "errorIndexingBolt" + grouping: + streamId: "error" + type: SHUFFLE diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/FalconHoseExampleOutput b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/FalconHoseExampleOutput new file mode 100644 index 0000000000..8f9842d0cf --- /dev/null +++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/FalconHoseExampleOutput @@ -0,0 +1,4 @@ +{"metadata":{"offset":1,"eventType":"LoginAuditEvent"},"event":{"LoginTime":1447097764168,"UserId":"user1@example.com","UserIp":"192.168.0.50","OperationName":"UserAuthenticate","ServiceName":"TokenApi","Success":true}} +{"metadata":{"offset":2,"eventType":"LoginAuditEvent"},"event":{"LoginTime":1447097785969,"UserId":"user1@example.com","UserIp":"192.168.0.50","OperationName":"UserAuthenticate","ServiceName":"TokenApi","Success":true}} +{"metadata":{"offset":3,"eventType":"DetectionSummaryEvent"},"event":{"ProcessStartTime":1447097902,"ProcessEndTime":1447097905,"ProcessId":12345,"ParentProcessId":23456,"ComputerName":"fake-computer-1","UserName":"user2","DetectName":"Suspicious Activity","DetectDescription":"An administrative/reconnaissance tool was spawned under an IIS worker process","Severity":2,"SeverityName":"Low","FileName":"regsvr32.exe","FilePath":"\\Device\\HarddiskVolume1\\Windows\\SysWOW64","CommandLine":"regsvr32.exe /u /s C:\\Windows\\system32\\dxtmsft.dll","SHA256String":"890c1734ed1ef6b2422a9b21d6205cf91e014add8a7f41aa5a294fcf60631a7b","MD5String":"432be6cf7311062633459eef6b242fb5","SHA1String":"N/A","MachineDomain":"fake-domain-1","FalconHostLink":"https://falcon.crowdstrike.com/detects/123","SensorId":"6789"}} +{"metadata":{"offset":4,"eventType":"DetectionSummaryEvent"},"event":{"ProcessStartTime":1447097921,"ProcessEndTime":1447097921,"ProcessId":34567,"ParentProcessId":45678,"ComputerName":"fake-computer-2","UserName":"user2","DetectName":"Suspicious Activity","DetectDescription":"An administrative/reconnaissance tool was spawned under an IIS worker process","Severity":2,"SeverityName":"Low","FileName":"regsvr32.exe","FilePath":"\\Device\\HarddiskVolume1\\Windows\\SysWOW64","CommandLine":"regsvr32.exe /s C:\\Windows\\system32\\dxtrans.dll","SHA256String":"890c1734ed1ef6b2422a9b21d6205cf91e014add8a7f41aa5a294fcf60631a7b","MD5String":"432be6cf7311062633459eef6b242fb5","SHA1String":"N/A","MachineDomain":"fake-domain-2","FalconHostLink":"https://falcon.crowdstrike.com/detects/124","SensorId":"5678"}}