From 01bae0f688d5f95bba2f131823e705a7050157f2 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Mon, 23 Jan 2017 15:47:14 -0500 Subject: [PATCH 01/17] METRON-671: Refactor existing Ansible deployment to use Ambari MPack --- .../amazon-ec2/conf/defaults.yml | 7 - metron-deployment/amazon-ec2/playbook.yml | 10 +- metron-deployment/amazon-ec2/run.sh | 2 +- .../extra_modules/ambari_cluster_state.py | 2 +- .../full-dev-platform/group_vars/all | 6 - .../inventory/full-dev-platform/hosts | 20 +- .../inventory/metron_example/group_vars/all | 7 - .../inventory/metron_example/hosts | 25 +- .../quick-dev-platform/group_vars/all | 6 - .../2.3.3/configuration/elastic-env.xml | 15 +- .../2.3.3/configuration/elastic-site.xml | 8 + .../2.3.3/configuration/elastic-sysconfig.xml | 2 +- .../ELASTICSEARCH/2.3.3/metainfo.xml | 2 +- .../2.3.3/package/scripts/elastic.py | 8 +- .../2.3.3/package/scripts/params.py | 3 +- .../2.3.3/package/scripts/slave.py | 7 +- .../templates/elasticsearch.master.yaml.j2 | 4 +- .../KIBANA/4.5.1/configuration/kibana-env.xml | 19 +- .../4.5.1/configuration/kibana-site.xml | 2 +- .../4.5.1/package/scripts/kibana_master.py | 8 +- .../KIBANA/4.5.1/package/scripts/params.py | 4 +- .../CURRENT/configuration/metron-env.xml | 14 +- .../package/scripts/enrichment_commands.py | 5 +- .../package/scripts/enrichment_master.py | 7 - .../package/scripts/indexing_commands.py | 2 +- .../package/scripts/params/params_linux.py | 2 +- .../playbooks/ambari_install.yml | 14 + .../playbooks/metron_install.yml | 77 +-- .../roles/ambari_common/defaults/main.yml | 3 +- .../roles/ambari_config/meta/main.yml | 1 + .../roles/ambari_config/tasks/main.yml | 1 - .../roles/ambari_config/tasks/start_hdp.yml | 43 -- .../vars/multi_vagrant_cluster.yml | 99 --- .../ambari_config/vars/single_node_vm.yml | 32 +- .../ambari_config/vars/small_cluster.yml | 54 +- .../roles/ambari_gather_facts/tasks/main.yml | 32 +- .../roles/ambari_master/tasks/main.yml | 14 + .../meta/main.yml | 3 +- .../roles/deployment-report/tasks/main.yml | 1 - .../roles/elasticsearch/defaults/main.yml | 22 - .../elasticsearch/files/elasticsearch.repo | 24 - .../roles/elasticsearch/meta/main.yml | 24 - .../elasticsearch/tasks/elasticsearch.yml | 73 --- .../metron-elasticsearch-logrotate.yml | 27 - .../roles/java_jdk/defaults/main.yml | 2 +- .../roles/kafka-broker/defaults/main.yml | 18 - .../roles/kafka-broker/meta/main.yml | 18 - .../roles/kafka-broker/tasks/main.yml | 41 -- metron-deployment/roles/kibana/README.md | 35 -- .../roles/kibana/defaults/main.yml | 22 - .../roles/kibana/tasks/dashboard.yml | 22 - .../roles/kibana/tasks/dependencies.yml | 27 - .../roles/kibana/tasks/elasticdump.yml | 35 -- .../roles/kibana/tasks/kibana.yml | 47 -- metron-deployment/roles/kibana/tasks/main.yml | 21 - .../roles/kibana/templates/kibana-index.json | 34 - .../kibana/templates/kibana.repo.template | 22 - .../meta/main.yml | 0 .../roles/load_web_templates/tasks/main.yml | 32 + .../tasks/main.yml | 20 +- .../roles/metron_common/defaults/main.yml | 19 - .../roles/metron_common/tasks/main.yml | 36 -- .../files/es_templates/bro_index.template | 218 ------- .../files/es_templates/snort_index.template | 183 ------ .../files/es_templates/yaf_index.template | 205 ------ .../tasks/load_templates.yml | 53 -- .../tasks/main.yml | 18 - .../metron_hbase_tables/defaults/main.yml | 22 - .../roles/metron_hbase_tables/tasks/main.yml | 24 - .../metron_kafka_topics/defaults/main.yml | 30 - .../roles/metron_kafka_topics/meta/main.yml | 19 - .../roles/metron_kafka_topics/tasks/main.yml | 30 - .../roles/monit/defaults/main.yml | 3 - metron-deployment/roles/monit/tasks/main.yml | 3 +- .../roles/monit/tasks/monit-definitions.yml | 20 - .../roles/monit/tasks/scripts.yml | 55 -- .../monit/templates/monit/elasticsearch.monit | 23 - .../monit/templates/monit/enrichment.monit | 26 - .../monit/indexing-elasticsearch.monit | 26 - .../monit/templates/monit/indexing-solr.monit | 26 - .../roles/monit/templates/monit/kibana.monit | 22 - .../roles/monit/templates/monit/parsers.monit | 49 -- .../scripts/start_enrichment_topology.sh | 25 - .../monit/templates/scripts/start_topology.sh | 25 - .../templates/scripts/status_topology.sh | 33 - .../monit/templates/scripts/stop_topology.sh | 23 - .../roles/solr/defaults/main.yml | 29 - metron-deployment/roles/solr/files/schema.xml | 191 ------ metron-deployment/roles/solr/meta/main.yml | 21 - metron-deployment/roles/solr/tasks/main.yml | 74 --- .../roles/solr/templates/solr.xml | 52 -- .../roles/solr/templates/solrconfig.xml | 583 ------------------ .../vagrant/full-dev-platform/Vagrantfile | 2 +- 93 files changed, 296 insertions(+), 3009 deletions(-) delete mode 100644 metron-deployment/roles/ambari_config/tasks/start_hdp.yml delete mode 100644 metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml rename metron-deployment/roles/{metron_common => ambari_slave}/meta/main.yml (97%) delete mode 100644 metron-deployment/roles/elasticsearch/defaults/main.yml delete mode 100644 metron-deployment/roles/elasticsearch/files/elasticsearch.repo delete mode 100644 metron-deployment/roles/elasticsearch/meta/main.yml delete mode 100644 metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml delete mode 100644 metron-deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml delete mode 100644 metron-deployment/roles/kafka-broker/defaults/main.yml delete mode 100644 metron-deployment/roles/kafka-broker/meta/main.yml delete mode 100644 metron-deployment/roles/kafka-broker/tasks/main.yml delete mode 100644 metron-deployment/roles/kibana/README.md delete mode 100644 metron-deployment/roles/kibana/defaults/main.yml delete mode 100644 metron-deployment/roles/kibana/tasks/dashboard.yml delete mode 100644 metron-deployment/roles/kibana/tasks/dependencies.yml delete mode 100644 metron-deployment/roles/kibana/tasks/elasticdump.yml delete mode 100644 metron-deployment/roles/kibana/tasks/kibana.yml delete mode 100644 metron-deployment/roles/kibana/tasks/main.yml delete mode 100644 metron-deployment/roles/kibana/templates/kibana-index.json delete mode 100644 metron-deployment/roles/kibana/templates/kibana.repo.template rename metron-deployment/roles/{metron_hbase_tables => load_web_templates}/meta/main.yml (100%) create mode 100644 metron-deployment/roles/load_web_templates/tasks/main.yml rename metron-deployment/roles/{elasticsearch => metron-rpms}/tasks/main.yml (72%) delete mode 100644 metron-deployment/roles/metron_common/defaults/main.yml delete mode 100644 metron-deployment/roles/metron_common/tasks/main.yml delete mode 100644 metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/bro_index.template delete mode 100644 metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/snort_index.template delete mode 100644 metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template delete mode 100644 metron-deployment/roles/metron_elasticsearch_templates/tasks/load_templates.yml delete mode 100644 metron-deployment/roles/metron_elasticsearch_templates/tasks/main.yml delete mode 100644 metron-deployment/roles/metron_hbase_tables/defaults/main.yml delete mode 100644 metron-deployment/roles/metron_hbase_tables/tasks/main.yml delete mode 100644 metron-deployment/roles/metron_kafka_topics/defaults/main.yml delete mode 100644 metron-deployment/roles/metron_kafka_topics/meta/main.yml delete mode 100644 metron-deployment/roles/metron_kafka_topics/tasks/main.yml delete mode 100644 metron-deployment/roles/monit/tasks/scripts.yml delete mode 100644 metron-deployment/roles/monit/templates/monit/elasticsearch.monit delete mode 100644 metron-deployment/roles/monit/templates/monit/enrichment.monit delete mode 100644 metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit delete mode 100644 metron-deployment/roles/monit/templates/monit/indexing-solr.monit delete mode 100644 metron-deployment/roles/monit/templates/monit/kibana.monit delete mode 100644 metron-deployment/roles/monit/templates/monit/parsers.monit delete mode 100644 metron-deployment/roles/monit/templates/scripts/start_enrichment_topology.sh delete mode 100644 metron-deployment/roles/monit/templates/scripts/start_topology.sh delete mode 100644 metron-deployment/roles/monit/templates/scripts/status_topology.sh delete mode 100644 metron-deployment/roles/monit/templates/scripts/stop_topology.sh delete mode 100644 metron-deployment/roles/solr/defaults/main.yml delete mode 100644 metron-deployment/roles/solr/files/schema.xml delete mode 100644 metron-deployment/roles/solr/meta/main.yml delete mode 100644 metron-deployment/roles/solr/tasks/main.yml delete mode 100644 metron-deployment/roles/solr/templates/solr.xml delete mode 100644 metron-deployment/roles/solr/templates/solrconfig.xml diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml index 84dbcce814..c9b435b181 100644 --- a/metron-deployment/amazon-ec2/conf/defaults.yml +++ b/metron-deployment/amazon-ec2/conf/defaults.yml @@ -17,19 +17,12 @@ --- # which services should be started? services_to_start: - - elasticsearch - pcap-service - - kibana - yaf - snort - snort-logs - bro - pcap-replay - - yaf-parser - - bro-parser - - snort-parser - - enrichment - - indexing # the ami for cent6 by region amis_by_region: diff --git a/metron-deployment/amazon-ec2/playbook.yml b/metron-deployment/amazon-ec2/playbook.yml index a0b15dce09..fb19d89d1e 100644 --- a/metron-deployment/amazon-ec2/playbook.yml +++ b/metron-deployment/amazon-ec2/playbook.yml @@ -26,12 +26,12 @@ - include: tasks/create-vpc.yml - include: tasks/create-open-inbound-security-group.yml - include: tasks/create-open-outbound-security-group.yml - - include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,metron,ec2 + - include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,ec2,monit - include: tasks/create-hosts.yml host_count=4 host_type=ambari_slave,ec2 - - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,metron_kafka_topics,metron_hbase_tables,metron,ec2,pcap_server - - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2 - - include: tasks/create-hosts.yml host_count=2 host_type=search,metron,ec2 - - include: tasks/create-hosts.yml host_count=1 host_type=web,metron,ec2 + - include: tasks/create-hosts.yml host_count=1 host_type=pcap_server,monit,ec2 + - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2,zeppelin + - include: tasks/create-hosts.yml host_count=2 host_type=ambari_slave,search,ec2 + - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,web,ec2 tags: - ec2 diff --git a/metron-deployment/amazon-ec2/run.sh b/metron-deployment/amazon-ec2/run.sh index cad4fb7389..cea45b38df 100755 --- a/metron-deployment/amazon-ec2/run.sh +++ b/metron-deployment/amazon-ec2/run.sh @@ -67,6 +67,6 @@ RC=$?; if [[ $RC != 0 ]]; then exit $RC; fi cd $DEPLOYDIR export EC2_INI_PATH=conf/ec2.ini ansible-playbook -i ec2.py playbook.yml \ - --skip-tags="solr, sensor-stubs" \ + --skip-tags="sensor-stubs" \ --extra-vars="env=$ENV" \ $EXTRA_ARGS diff --git a/metron-deployment/extra_modules/ambari_cluster_state.py b/metron-deployment/extra_modules/ambari_cluster_state.py index a61d0f1123..691d48edfd 100644 --- a/metron-deployment/extra_modules/ambari_cluster_state.py +++ b/metron-deployment/extra_modules/ambari_cluster_state.py @@ -385,9 +385,9 @@ def blueprint_var_to_ambari_converter(blueprint_var): this_host_map['hosts'] = this_host_list host_map.append(this_host_map) blueprint = dict() + blueprint['configurations'] = blueprint_var['required_configurations'] blueprint['host_groups'] = new_groups blueprint['Blueprints'] = {'stack_name': blueprint_var['stack_name'], 'stack_version': blueprint_var['stack_version']} - return blueprint, host_map from ansible.module_utils.basic import * diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all index 2b4617f608..56ad75d63e 100644 --- a/metron-deployment/inventory/full-dev-platform/group_vars/all +++ b/metron-deployment/inventory/full-dev-platform/group_vars/all @@ -17,17 +17,11 @@ # which services should be started? services_to_start: - - elasticsearch - pcap-service - - kibana - snort - snort-logs - bro - pcap-replay - - bro-parser - - snort-parser - - enrichment - - indexing # ambari ambari_host: "{{ groups.ambari_master[0] }}" diff --git a/metron-deployment/inventory/full-dev-platform/hosts b/metron-deployment/inventory/full-dev-platform/hosts index 98e8681ea9..54a549b368 100644 --- a/metron-deployment/inventory/full-dev-platform/hosts +++ b/metron-deployment/inventory/full-dev-platform/hosts @@ -21,32 +21,24 @@ node1 [ambari_slave] node1 -[metron_hbase_tables] +[metron] node1 -[metron_kafka_topics] +[search] node1 -[enrichment] +[sensors] node1 -[search] +[pcap_server] node1 [web] node1 -[sensors] -node1 - -[pcap_server] +[zeppelin] node1 -[metron:children] -enrichment -search -web +[monit:children] sensors -metron_kafka_topics -metron_hbase_tables pcap_server diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all index aeefa447db..614925df50 100644 --- a/metron-deployment/inventory/metron_example/group_vars/all +++ b/metron-deployment/inventory/metron_example/group_vars/all @@ -17,19 +17,12 @@ --- # which services should be started? services_to_start: - - elasticsearch - pcap-service - - kibana - yaf - snort - snort-logs - bro - pcap-replay - - yaf-parser - - bro-parser - - snort-parser - - enrichment - - indexing #Ansible Variables ansible_ssh_private_key_file: /Path/to/private/key/file #Change This diff --git a/metron-deployment/inventory/metron_example/hosts b/metron-deployment/inventory/metron_example/hosts index c53466de93..8f09e4e462 100644 --- a/metron-deployment/inventory/metron_example/hosts +++ b/metron-deployment/inventory/metron_example/hosts @@ -28,18 +28,9 @@ node6 node7 node8 -[metron_hbase_tables] -node9 - -[metron_kafka_topics] -node9 - -[pcap_server] -node9 - #3rd ambari_slave -[enrichment] -node1 +[metron] +node4 #1 or more [search] @@ -47,17 +38,15 @@ node10 node11 node12 -#1 only [sensors] node1 +[pcap_server] +node9 + [web] node12 -[metron:children] -enrichment -search -web +[monit:children] sensors -metron_kafka_topics -metron_hbase_tables +pcap_server diff --git a/metron-deployment/inventory/quick-dev-platform/group_vars/all b/metron-deployment/inventory/quick-dev-platform/group_vars/all index d62e74ba1b..e835d79762 100644 --- a/metron-deployment/inventory/quick-dev-platform/group_vars/all +++ b/metron-deployment/inventory/quick-dev-platform/group_vars/all @@ -17,17 +17,11 @@ # which services should be started? services_to_start: - - elasticsearch - pcap-service - - kibana - snort - snort-logs - bro - pcap-replay - - bro-parser - - snort-parser - - enrichment - - indexing # ambari ambari_host: "{{ groups.ambari_master[0] }}" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml index 2939c284a0..a4de039abd 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml @@ -23,12 +23,21 @@ elastic_user elasticsearch USER - The user for Elasticsearch + Service user for Elasticsearch + + user + false + - user_group + elastic_group elasticsearch - The group for Elasticsearch + GROUP + Service group for Elasticsearch + + user + false + elastic_log_dir diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml index 3e7b6a56e6..36850907f1 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml @@ -26,6 +26,14 @@ metron Cluster name identifies your cluster + + masters_also_are_datanodes + "false" + ES Masters and Slaves cannot be installed on the same nodes. Set this to true if you want the ES master nodes to serve as combined master/datanodes. + + string + + zen_discovery_ping_unicast_hosts diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml index 58e4916062..6ef09ac290 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml @@ -90,7 +90,7 @@ CONF_DIR={{conf_dir}} ES_USER={{elastic_user}} # Additional Java OPTS -ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled +ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml index 25f98e58e9..53909a2bf5 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml @@ -39,7 +39,7 @@ ES_SLAVE Elasticsearch Data Node SLAVE - 3+ + 0+ PYTHON diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py index a4726557bb..2e638de0c8 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py @@ -22,12 +22,12 @@ from resource_management.core.resources.system import File from resource_management.core.source import InlineTemplate from resource_management.core.source import Template - +from resource_management.core.resources import User def elastic(): - print "INSIDE THE %s" % __file__ import params + User(params.elastic_user, action = "create", groups = params.elastic_group) params.path_data = params.path_data.replace('"', '') data_path = params.path_data.replace(' ', '').split(',') data_path[:] = [x.replace('"', '') for x in data_path] @@ -37,15 +37,15 @@ def elastic(): Directory(directories, create_parents=True, - # recursive=True, mode=0755, owner=params.elastic_user, - group=params.elastic_user + group=params.elastic_group ) print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir) File("{0}/elastic-env.sh".format(params.conf_dir), owner=params.elastic_user, + group=params.elastic_group, content=InlineTemplate(params.elastic_env_sh_template) ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py index 8b890d4ff1..aea81fbaca 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py @@ -23,6 +23,7 @@ # server configurations config = Script.get_config() +masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes'] elastic_home = config['configurations']['elastic-sysconfig']['elastic_home'] data_dir = config['configurations']['elastic-sysconfig']['data_dir'] work_dir = config['configurations']['elastic-sysconfig']['work_dir'] @@ -32,7 +33,7 @@ max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] elastic_user = config['configurations']['elastic-env']['elastic_user'] -user_group = config['configurations']['elastic-env']['user_group'] +elastic_group = config['configurations']['elastic-env']['elastic_group'] log_dir = config['configurations']['elastic-env']['elastic_log_dir'] pid_dir = '/var/run/elasticsearch' pid_file = '/var/run/elasticsearch/elasticsearch.pid' diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py index 90d4d3d245..15a5cf50f0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py @@ -22,11 +22,12 @@ from resource_management.core.resources.system import File from resource_management.core.source import InlineTemplate from resource_management.core.source import Template - +from resource_management.core.resources import User def slave(): import params + User(params.elastic_user, action = "create", groups = params.elastic_group) params.path_data = params.path_data.replace('"', '') data_path = params.path_data.replace(' ', '').split(',') data_path[:] = [x.replace('"', '') for x in data_path] @@ -38,7 +39,7 @@ def slave(): create_parents=True, mode=0755, owner=params.elastic_user, - group=params.elastic_user, + group=params.elastic_group, cd_access="a" ) @@ -54,7 +55,7 @@ def slave(): "elasticsearch.slave.yaml.j2", configurations=configurations), owner=params.elastic_user, - group=params.elastic_user + group=params.elastic_group ) print "Master sysconfig: /etc/sysconfig/elasticsearch" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 index 8d92f1b88e..ca3630a5ae 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 @@ -28,12 +28,12 @@ discovery: zen: ping: multicast: - enabled: {{discovery_zen_ping_multicast_enabled}} + enabled: {{ discovery_zen_ping_multicast_enabled }} unicast: hosts: "{{zen_discovery_ping_unicast_hosts}}" node: - data: false + data: {{ masters_also_are_datanodes }} master: true name: {{hostname}} path: diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml index 972945dfba..0adf32ad7a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml @@ -23,13 +23,21 @@ kibana_user kibana USER - + Service User for Kibana + + user + false + - user_group + kabana_group kibana GROUP - + Service Group for Kibana + + user + false + kibana_log_dir @@ -51,4 +59,9 @@ 5000 Kibana back end server port to use. + + kibana_default_application + default + The default application to load. + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml index c546e2c2ba..4373d140f9 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml @@ -50,7 +50,7 @@ elasticsearch.url: {{ es_url }} # kibana.index: ".kibana" # The default application to load. -# kibana.defaultAppId: "discover" +kibana.defaultAppId: "{{ kibana_default_application }}" # If your Elasticsearch is protected with basic auth, these are the user credentials # used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py index f296272746..028ea9ea5c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py @@ -53,9 +53,9 @@ def install(self, env): majorVersion = OSCheck.get_os_major_version() Logger.info("CentOS/RHEL major version reported by Ambari: " + majorVersion) if majorVersion == "6" or majorVersion == "7": - repoName = "name=CentOS/RHEL {} repository for Elasticsearch Curator 4.x packages\n".format(majorVersion) - baseUrl = "baseurl=http://packages.elastic.co/curator/4/centos/{}\n".format(majorVersion) - Logger.info("Installing Elasticsearch Curator CentOS/RHEL {} repo".format(majorVersion)) + repoName = "name=CentOS/RHEL {0} repository for Elasticsearch Curator 4.x packages\n".format(majorVersion) + baseUrl = "baseurl=http://packages.elastic.co/curator/4/centos/{0}\n".format(majorVersion) + Logger.info("Installing Elasticsearch Curator CentOS/RHEL {0} repo".format(majorVersion)) Execute("echo \"[curator-4]\n" + repoName + baseUrl + @@ -81,7 +81,7 @@ def configure(self, env, upgrade_type=None, config_dir=None): group=params.kibana_user ) - File("{}/kibana.yml".format(params.conf_dir), + File("{0}/kibana.yml".format(params.conf_dir), owner=params.kibana_user, content=InlineTemplate(params.kibana_yml_template) ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py index ef1e597ae0..a5e0ec96c3 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py @@ -33,7 +33,7 @@ conf_dir = "/opt/kibana/config/" kibana_user = config['configurations']['kibana-env']['kibana_user'] -user_group = config['configurations']['kibana-env']['user_group'] +kibana_group = config['configurations']['kibana-env']['kibana_group'] log_dir = config['configurations']['kibana-env']['kibana_log_dir'] pid_dir = config['configurations']['kibana-env']['kibana_pid_dir'] pid_file = format("{pid_dir}/kibanasearch.pid") @@ -42,6 +42,8 @@ es_host = parsed.netloc.split(':')[0] es_port = parsed.netloc.split(':')[1] kibana_port = config['configurations']['kibana-env']['kibana_server_port'] +kibana_default_application = config['configurations']['kibana-env']['kibana_default_application'] hostname = config['hostname'] java64_home = config['hostLevelParams']['java_home'] kibana_yml_template = config['configurations']['kibana-site']['content'] + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index eeb20372db..16bc25e168 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -46,15 +46,23 @@ metron_user metron USER - The user for Metron - Metron User + Service User for Metron + Metron User Name + + user + false + metron_group metron GROUP - The group for Metron + Service Group for Metron Metron Group Name + + user + false + metron_topic_retention diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index bc73c87eb3..87c78e23f2 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -46,6 +46,7 @@ def is_configured(self): return self.__configured def set_configured(self): + Logger.info("Setting Configured to True") File(self.__params.enrichment_configured_flag_file, content="", owner=self.__params.metron_user, @@ -88,7 +89,7 @@ def init_geo(self): type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.user_group, + group=self.__params.metron_group, mode=0775, ) @@ -180,6 +181,7 @@ def is_topology_active(self, env): return active def create_hbase_tables(self): + Logger.info("Creating HBase Tables") add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf) Execute(add_enrichment_cmd, tries=3, @@ -195,3 +197,4 @@ def create_hbase_tables(self): logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' ) + Logger.info("Done creating HBase Tables") diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index b02ba5cdc7..71d80c88a7 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -48,7 +48,6 @@ def start(self, env, upgrade_type=None): from params import params env.set_params(params) commands = EnrichmentCommands(params) - metron_service.load_global_config(params) if not commands.is_configured(): @@ -79,12 +78,6 @@ def restart(self, env): commands = EnrichmentCommands(params) commands.restart_enrichment_topology(env) - def kafkabuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = EnrichmentCommands(params) - commands.init_kafka_topics() - if __name__ == "__main__": Enrichment().execute() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 8ed3f9a734..50af3b2361 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -101,7 +101,7 @@ def init_hdfs_dir(self): type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.user_group, + group=self.__params.metron_group, mode=0775, ) Logger.info('Done creating HDFS indexing directory') diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 2427d255f9..a0446e417b 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -39,7 +39,7 @@ tmp_dir = Script.get_tmp_dir() hostname = config['hostname'] -user_group = config['configurations']['cluster-env']['user_group'] +metron_group = config['configurations']['cluster-env']['metron_group'] metron_home = status_params.metron_home parsers = status_params.parsers geoip_url = config['configurations']['metron-env']['geoip_url'] diff --git a/metron-deployment/playbooks/ambari_install.yml b/metron-deployment/playbooks/ambari_install.yml index a8f3f7e451..51aa9071b9 100644 --- a/metron-deployment/playbooks/ambari_install.yml +++ b/metron-deployment/playbooks/ambari_install.yml @@ -54,6 +54,13 @@ - ambari-agent - hdp-install +- hosts: metron + become: true + roles: + - role: metron-rpms + tags: + - metron-deploy + - hosts: ambari_master become: true roles: @@ -61,3 +68,10 @@ tags: - hdp-install - hdp-deploy + +- hosts: ambari_master + become: true + roles: + - role: load_web_templates + tags: + - load_templates diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml index 359808a0f4..6fc95e3a49 100644 --- a/metron-deployment/playbooks/metron_install.yml +++ b/metron-deployment/playbooks/metron_install.yml @@ -29,65 +29,6 @@ tags: - packer -# -# prerequisites -# -- hosts: metron - become: true - roles: - - role: metron_common - tags: - - metron-prereqs - -# -# search -# -- hosts: search - become: true - vars: - es_hosts: "{% set comma = joiner(',') %}{% for host in groups['search'] -%}{{ comma() }}{{ host }}{%- endfor %}" - roles: - - role: elasticsearch - - { role: metron_elasticsearch_templates, tags: ['load_es_templates'] } - tags: - - elasticsearch - -- hosts: search - become: true - roles: - - role: solr - tags: - - solr - -# -# parsers, enrichment, and indexing topologies -# -- hosts: metron_hbase_tables - become: true - roles: - - role: ambari_gather_facts - - role: metron_hbase_tables - tags: - - metron-hbase-tables - -- hosts: metron_kafka_topics - become: true - roles: - - role: ambari_gather_facts - - role: metron_kafka_topics - tags: - - metron-kafka-topics - -- hosts: enrichment - become: true - tags: - - enrichment - roles: - - role: metron_streaming - -# -# user interface -# - hosts: pcap_server become: true roles: @@ -95,13 +36,6 @@ tags: - pcap-service -- hosts: web - become: true - roles: - - role: kibana - tags: - - kibana - # # sensors # @@ -131,10 +65,17 @@ tags: - sensor-stubs +- hosts: metron + become: true + roles: + - role: metron-rpms + tags: + - copy-rpms + # # monitor and start metron services with monit # -- hosts: metron +- hosts: monit become: true roles: - { role: ambari_gather_facts, tags: ['always'] } @@ -144,7 +85,7 @@ # # deployment report # -- hosts: metron +- hosts: monit become: false roles: - { role: deployment-report, tags: ['report'] } diff --git a/metron-deployment/roles/ambari_common/defaults/main.yml b/metron-deployment/roles/ambari_common/defaults/main.yml index eda4a06417..2c9389c63e 100644 --- a/metron-deployment/roles/ambari_common/defaults/main.yml +++ b/metron-deployment/roles/ambari_common/defaults/main.yml @@ -17,5 +17,4 @@ --- hadoop_logrotate_frequency: daily hadoop_logrotate_retention: 30 - -rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.0.1/ambari.repo" \ No newline at end of file +rhel_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo diff --git a/metron-deployment/roles/ambari_config/meta/main.yml b/metron-deployment/roles/ambari_config/meta/main.yml index 61197e3e1a..8f65a28ebc 100644 --- a/metron-deployment/roles/ambari_config/meta/main.yml +++ b/metron-deployment/roles/ambari_config/meta/main.yml @@ -19,3 +19,4 @@ dependencies: - epel - python-pip - httplib2 + - java_jdk diff --git a/metron-deployment/roles/ambari_config/tasks/main.yml b/metron-deployment/roles/ambari_config/tasks/main.yml index 7697202dda..aa1004c48e 100644 --- a/metron-deployment/roles/ambari_config/tasks/main.yml +++ b/metron-deployment/roles/ambari_config/tasks/main.yml @@ -46,5 +46,4 @@ wait_for_complete: True blueprint_var: "{{ blueprint }}" -- include: start_hdp.yml diff --git a/metron-deployment/roles/ambari_config/tasks/start_hdp.yml b/metron-deployment/roles/ambari_config/tasks/start_hdp.yml deleted file mode 100644 index 36f797f8fd..0000000000 --- a/metron-deployment/roles/ambari_config/tasks/start_hdp.yml +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# -# Workaround for Kafka not starting -# Fire off async start followed by -# Sync start -execution will pause until -# final start completes. -# -- name: Start the ambari cluster - no wait - ambari_cluster_state: - host: "{{ groups.ambari_master[0] }}" - port: "{{ ambari_port }}" - username: "{{ ambari_user }}" - password: "{{ ambari_password }}" - cluster_name: "{{ cluster_name }}" - cluster_state: started - wait_for_complete: False - -- name: Start the ambari cluster - wait - ambari_cluster_state: - host: "{{ groups.ambari_master[0] }}" - port: "{{ ambari_port }}" - username: "{{ ambari_user }}" - password: "{{ ambari_password }}" - cluster_name: "{{ cluster_name }}" - cluster_state: started - wait_for_complete: True - diff --git a/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml b/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml deleted file mode 100644 index 18d3b8d1f5..0000000000 --- a/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml +++ /dev/null @@ -1,99 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- - -hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER] -hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, NODEMANAGER] -spark_master: [SPARK_JOBHISTORYSERVER] -storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER] -storm_slave: [SUPERVISOR] -kafka_broker: [KAFKA_BROKER] -zookeeper_master: [ZOOKEEPER_SERVER] -hbase_master: [HBASE_MASTER] -hbase_slave: [HBASE_REGIONSERVER] -hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT] - -master_1_components: "{{ hadoop_master | union(hadoop_clients) }}" - -master_1_host: - - "{{groups.ambari_slave[0]}}" -master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) }}" -master_2_host: - - "{{groups.ambari_slave[1]}}" -slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}" - -cluster_name: "metron" -blueprint_name: "metron_blueprint" - -configurations: - - zoo.cfg: - dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}' - - hdfs-site: - dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}' - dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}' - dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}' - dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}' - - hadoop-env: - namenode_heapsize: 2048 - dtnode_heapsize: 1024 - - hbase-env: - hbase_regionserver_heapsize: 1024 - hbase_master_heapsize: 1024 - - yarn-env: - nodemanager_heapsize: 512 - yarn_heapsize: 512 - apptimelineserver_heapsize : 512 - - mapred-env: - jobhistory_heapsize: 256 - - yarn-site: - yarn.nodemanager.resource.memory-mb: 1024 - yarn.scheduler.maximum-allocation-mb: 1024 - yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}' - yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}' - yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}' - yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}' - - - mapred-site: - mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}' - - storm-site: - supervisor.slots.ports: "[6700, 6701, 6702, 6703]" - storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}' - topology.classpath: '{{ topology_classpath }}' - - kafka-env: - content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}" - - kafka-broker: - log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}' - -blueprint: - stack_name: HDP - stack_version: "{{ hdp_stack }}" - groups: - - name : master_1 - cardinality: 1 - configuration: [] # configuration not yet implemented - components: "{{ master_1_components }}" - hosts: "{{ master_1_host }}" - - name : master_2 - cardinality: 1 - configuration: [] # configuration not yet implemented - components: "{{ master_2_components }}" - hosts: "{{ master_2_host }}" - - name: slaves - cardinality: 1+ - configuration: [] # configuration not yet implemented - components: "{{ slave_components }}" - hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) }}" diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml index 429981e765..46a486eb38 100644 --- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml @@ -28,8 +28,16 @@ zookeeper_master: [ZOOKEEPER_SERVER] zookeeper_slave: [ZOOKEEPER_CLIENT] hbase_master: [HBASE_MASTER, HBASE_CLIENT] hbase_slave: [HBASE_REGIONSERVER] +es_master: [ES_MASTER] +kibana_master: [KIBANA_MASTER] +metron_indexing: [METRON_INDEXING] +metron_enrichment_master : [METRON_ENRICHMENT_MASTER] +metron_parsers : [METRON_PARSERS] -metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(kafka_broker) | union(hbase_slave) }}" +metron_components: > + {{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | + union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | + union(metron_enrichment_master) | union(metron_parsers) | union(es_master) }} cluster_name: "metron_cluster" blueprint_name: "metron_blueprint" @@ -80,10 +88,32 @@ configurations: - kafka-broker: log.dirs: '{{ kafka_log_dirs }}' delete.topic.enable: "true" + - metron-env: + parsers: "bro,snort" + - elastic-site: + index_number_of_shards: 1 + index_number_of_replicas: 0 + zen_discovery_ping_unicast_hosts: "{{ groups.search | join(',') }}" + gateway_recover_after_data_nodes: 1 + network_host: "_lo_,_eth0_,_eth1_" + masters_also_are_datanodes: "1" + +required_configurations: + - metron-env: + storm_rest_addr: "{{ groups.ambari_slave[0] }}:8744" + es_hosts: "{{ groups.search | join(',') }}" + zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995" + - kibana-env: + kibana_pid_dir: /var/run/kibana + kibana_es_url: http://{{ groups.search[0] }}:9200 + kibana_log_dir: /var/log/kibana + kibana_server_port: 5000 + kibana_default_application: "dashboard/Metron-Dashboard" blueprint: stack_name: HDP stack_version: "{{ hdp_stack }}" + required_configurations: "{{ required_configurations }}" groups: - name : host_group_1 cardinality: 1 diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml index a976d007dc..2f1574037e 100644 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml @@ -17,7 +17,8 @@ --- hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER] -hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, NODEMANAGER] +app_timeline_server: [APP_TIMELINE_SERVER] +hadoop_slave: [DATANODE, NODEMANAGER] spark_master: [SPARK_JOBHISTORYSERVER] storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER] storm_slave: [SUPERVISOR] @@ -26,13 +27,27 @@ zookeeper_master: [ZOOKEEPER_SERVER] hbase_master: [HBASE_MASTER] hbase_slave: [HBASE_REGIONSERVER] hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT] +es_master: [ES_MASTER] +es_slave: [ES_SLAVE] +kibana_master: [KIBANA_MASTER] +metron_indexing: [METRON_INDEXING] +metron_enrichment_master : [METRON_ENRICHMENT_MASTER] +metron_parsers : [METRON_PARSERS] -master_1_components: "{{ hadoop_master | union(hadoop_clients) }}" +master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}" master_1_host: - "{{groups.ambari_slave[0]}}" -master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) }}" +master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}" master_2_host: - "{{groups.ambari_slave[1]}}" +metron_components: > + {{ metron_indexing | union(metron_enrichment_master) | union(metron_parsers) | union(hadoop_slave) | union(storm_slave) | + union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }} +metron_host: + - "{{ groups.metron[0] }}" +web_components: "{{ kibana_master | union(es_master) }}" +web_host: + - "{{ groups.web[0] }}" slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}" cluster_name: "metron" @@ -68,10 +83,31 @@ configurations: topology.classpath: '{{ topology_classpath }}' - kafka-broker: log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}' + - metron-env: + parsers: "bro,snort,yaf" + - elastic-site: + index_number_of_shards: 2 + index_number_of_replicas: 1 + zen_discovery_ping_unicast_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}" + gateway_recover_after_data_nodes: 1 + network_host: _lo_,_{{ elasticsearch_network_interface }}_ + +required_configurations: + - metron-env: + storm_rest_addr: "{{ groups.ambari_slave[1] }}:8744" + es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}" + zeppelin_server_url: "{{ groups.zeppelin[0] }}" + - kibana-env: + kibana_pid_dir: /var/run/kibana + kibana_es_url: http://{{ groups.web[0] }}:9200 + kibana_log_dir: /var/log/kibana + kibana_server_port: 5000 + kibana_default_application: "dashboard/Metron-Dashboard" blueprint: stack_name: HDP stack_version: "{{ hdp_stack }}" + required_configurations: "{{ required_configurations }}" groups: - name : master_1 cardinality: 1 @@ -83,8 +119,18 @@ blueprint: configuration: [] # configuration not yet implemented components: "{{ master_2_components }}" hosts: "{{ master_2_host }}" + - name : metron + cardinality: 1 + configuration: [] # configuration not yet implemented + components: "{{ metron_components }}" + hosts: "{{ metron_host }}" + - name : web + cardinality: 1 + configuration: [] # configuration not yet implemented + components: "{{ web_components }}" + hosts: "{{ web_host }}" - name: slaves cardinality: 1+ configuration: [] # configuration not yet implemented components: "{{ slave_components }}" - hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) }}" + hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}" diff --git a/metron-deployment/roles/ambari_gather_facts/tasks/main.yml b/metron-deployment/roles/ambari_gather_facts/tasks/main.yml index 69e8eb5b54..c976e278a8 100644 --- a/metron-deployment/roles/ambari_gather_facts/tasks/main.yml +++ b/metron-deployment/roles/ambari_gather_facts/tasks/main.yml @@ -193,6 +193,35 @@ zookeeper_url: "{% for host in zookeeper_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ zookeeper_port }}{% endfor %}" when: zookeeper_url is undefined +- name: "Ask Ambari: metron_hosts" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/METRON/components/METRON_INDEXING" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: metron_hosts_response + when: metron_hosts is undefined + +- set_fact: + metron_hosts: "{{ (metron_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" + when: metron_hosts is undefined + +- name: "Ask Ambari: kibana_hosts" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KIBANA/components/KIBANA_MASTER" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: kibana_hosts_response + when: kibana_hosts is undefined + +- set_fact: + kibana_hosts: "{{ (kibana_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" + when: kibana_hosts is undefined + + # # debug output # @@ -203,4 +232,5 @@ zookeeper_url = {{ zookeeper_url }}, kafka_broker_port = {{ kafka_broker_port }}, kafka_broker_hosts = {{ kafka_broker_hosts }}, - kafka_broker_url = {{ kafka_broker_url }}" + kafka_broker_url = {{ kafka_broker_url }}, + metron_hosts = {{ metron_hosts }}" diff --git a/metron-deployment/roles/ambari_master/tasks/main.yml b/metron-deployment/roles/ambari_master/tasks/main.yml index 05d321441d..935c48d90d 100644 --- a/metron-deployment/roles/ambari_master/tasks/main.yml +++ b/metron-deployment/roles/ambari_master/tasks/main.yml @@ -38,6 +38,20 @@ register: ambari_server_setup failed_when: ambari_server_setup.stderr +- name: Build MPack + local_action: shell cd {{ playbook_dir }}/.. && mvn clean package + become: false + +- name: Copy MPack to Ambari Host + copy: + src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-1.0.0.0-SNAPSHOT.tar.gz" + dest: /tmp + +- name: Install MPack on Ambari Host + shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-1.0.0.0-SNAPSHOT.tar.gz + args: + creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-1.0.0.0/addon-services + - name: start ambari server service: name: ambari-server diff --git a/metron-deployment/roles/metron_common/meta/main.yml b/metron-deployment/roles/ambari_slave/meta/main.yml similarity index 97% rename from metron-deployment/roles/metron_common/meta/main.yml rename to metron-deployment/roles/ambari_slave/meta/main.yml index 69ec2e1cb1..ddf6aa910d 100644 --- a/metron-deployment/roles/metron_common/meta/main.yml +++ b/metron-deployment/roles/ambari_slave/meta/main.yml @@ -16,5 +16,4 @@ # --- dependencies: - - epel - - ntp + - java_jdk diff --git a/metron-deployment/roles/deployment-report/tasks/main.yml b/metron-deployment/roles/deployment-report/tasks/main.yml index 3f947f36be..de76805d78 100644 --- a/metron-deployment/roles/deployment-report/tasks/main.yml +++ b/metron-deployment/roles/deployment-report/tasks/main.yml @@ -30,7 +30,6 @@ - " Metron @ http://{{ groups.web[0] }}:5000" - " Ambari @ http://{{ groups.ambari_master[0] }}:{{ ambari_port }}" - " Sensor Status @ http://{{ groups.sensors[0] }}:2812" - - " Topology Status @ http://{{ groups.enrichment[0] }}:2812" - " Zookeeper @ {{ zookeeper_url }}" - " Kafka @ {{ kafka_broker_url }}" - For additional information, see https://metron.incubator.apache.org/' diff --git a/metron-deployment/roles/elasticsearch/defaults/main.yml b/metron-deployment/roles/elasticsearch/defaults/main.yml deleted file mode 100644 index 002671760d..0000000000 --- a/metron-deployment/roles/elasticsearch/defaults/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -elasticsearch_data_dir: /var/lib/elasticsearch -elasticsearch_network_interface: eth0 -elasticsearch_logrotate_frequency: daily -elasticsearch_logrotate_retention: 30 - diff --git a/metron-deployment/roles/elasticsearch/files/elasticsearch.repo b/metron-deployment/roles/elasticsearch/files/elasticsearch.repo deleted file mode 100644 index 51dba70ac8..0000000000 --- a/metron-deployment/roles/elasticsearch/files/elasticsearch.repo +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -[elasticsearch-2.x] -name=Elasticsearch repository for 2.x packages -baseurl=https://packages.elastic.co/elasticsearch/2.x/centos -gpgcheck=1 -gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch -enabled=1 - diff --git a/metron-deployment/roles/elasticsearch/meta/main.yml b/metron-deployment/roles/elasticsearch/meta/main.yml deleted file mode 100644 index f5f059a63b..0000000000 --- a/metron-deployment/roles/elasticsearch/meta/main.yml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -dependencies: - - java_jdk - - epel - - python-pip - - httplib2 - - libselinux-python - diff --git a/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml b/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml deleted file mode 100644 index 92ef9c5949..0000000000 --- a/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Add Elasticsearch GPG key - rpm_key: - key: https://packages.elastic.co/GPG-KEY-elasticsearch - state: present - -- name: Add Elasticsearch repository - copy: - src: elasticsearch.repo - dest: /etc/yum.repos.d/elasticsearch.repo - mode: 0644 - -- name: Install Elasticsearch - yum: - name: elasticsearch - state: installed - register: result - until: result.rc == 0 - retries: 5 - delay: 10 - -- name: Create Data Directories - file: - path: "{{ item }}" - state: directory - mode: 0755 - owner: elasticsearch - group: elasticsearch - when: elasticsearch_data_dir is defined - with_items: - - '{{ elasticsearch_data_dir.split(",") }}' - -- name: Configure Elasticsearch - remove stale entries - lineinfile: > - dest=/etc/elasticsearch/elasticsearch.yml - regexp="{{ item }}" - state=absent - with_items: - - '^cluster\.name:' - - '^network\.host:' - - '^discovery\.zen\.ping\.unicast\.hosts:' - - '^path\.data' - -- name: Configure Elasticsearch - lineinfile: > - dest=/etc/elasticsearch/elasticsearch.yml - regexp="{{ item.regexp }}" - insertafter="{{ item.regexp }}" - line="{{ item.line }}" - state=present - with_items: - - { regexp: '^# *cluster\.name:', line: 'cluster.name: metron' } - - { regexp: '^# *network\.host:', line: 'network.host: ["_{{ - elasticsearch_network_interface }}:ipv4_","_local:ipv4_"]' } - - { regexp: '^# *discovery\.zen\.ping\.unicast\.hosts:', - line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'} - - { regexp: '^# *path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' } diff --git a/metron-deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml b/metron-deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml deleted file mode 100644 index c84f085621..0000000000 --- a/metron-deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#Elasticsearch -/var/log/elasticsearch/*.log { - {{ elasticsearch_logrotate_frequency }} - rotate {{ elasticsearch_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - diff --git a/metron-deployment/roles/java_jdk/defaults/main.yml b/metron-deployment/roles/java_jdk/defaults/main.yml index e593a71281..315bc0c720 100644 --- a/metron-deployment/roles/java_jdk/defaults/main.yml +++ b/metron-deployment/roles/java_jdk/defaults/main.yml @@ -15,4 +15,4 @@ # limitations under the License. # --- -java_home: /usr/jdk64/jdk1.8.0_60 \ No newline at end of file +java_home: /usr/jdk64/jdk1.8.0_77 diff --git a/metron-deployment/roles/kafka-broker/defaults/main.yml b/metron-deployment/roles/kafka-broker/defaults/main.yml deleted file mode 100644 index b0f5a11400..0000000000 --- a/metron-deployment/roles/kafka-broker/defaults/main.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo" diff --git a/metron-deployment/roles/kafka-broker/meta/main.yml b/metron-deployment/roles/kafka-broker/meta/main.yml deleted file mode 100644 index 9587e79417..0000000000 --- a/metron-deployment/roles/kafka-broker/meta/main.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: - - libselinux-python diff --git a/metron-deployment/roles/kafka-broker/tasks/main.yml b/metron-deployment/roles/kafka-broker/tasks/main.yml deleted file mode 100644 index db05cb0a45..0000000000 --- a/metron-deployment/roles/kafka-broker/tasks/main.yml +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Retrieve HDP repository definition - get_url: - url: "{{ hdp_repo_def }}" - dest: /etc/yum.repos.d/hdp.repo - mode: 0644 - -- name: Install kafka - yum: name={{item}} - with_items: - - java-1.8.0-openjdk - - kafka - - zookeeper-server - -- name: Create pid directories - file: path={{ item }} state=directory mode=0755 - with_items: - - /var/run/zookeeper - - /var/run/kafka - -- name: Start zookeeper - shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start - -- name: Start kafka - shell: /usr/hdp/current/kafka-broker/bin/kafka start diff --git a/metron-deployment/roles/kibana/README.md b/metron-deployment/roles/kibana/README.md deleted file mode 100644 index dd3571b3ec..0000000000 --- a/metron-deployment/roles/kibana/README.md +++ /dev/null @@ -1,35 +0,0 @@ -Kibana 4 --------- - -This role installs Kibana along with the default Metron Dashboard. - -### FAQ - -#### How do I change Metron's default dashboard? - -Kibana stores all configuration elements within an Elasticsearch index called `.kibana`. To deploy Kibana in a desired state, including the Metron Dashboard, we simply take an extract from a functioning Kibana instance and store that in `templates/kibana-index.json`. The deployment process then restores the index from this extract. - -(1) Stand-up an instance of Apache Metron and create the Kibana index patterns, visualizations, and dashboard as you see fit. - -(2) Run the following command to extract the definitions for all the components that you have created. Be sure to delete anything that you don't want to be part of this extract. It will include all artifacts present in your `.kibana` index. - - ``` - elasticdump --input=http://ec2-52-41-121-175.us-west-2.compute.amazonaws.com:9200/.kibana \ - --output=kibana-index.json \ - --type=data \ - --searchBody='{"filter": { "or": [ {"type": {"value": "search"}}, {"type": {"value":"dashboard"}}, {"type": {"value":"visualization"}},{"type": {"value": "config"}},{"type": {"value": "url"}},{"type": {"value": "index-pattern"}} ] }}' - ``` - -(3) This will result in a file containing the JSON-based definitions. Overwrite `templates/kibana-index.json`. - -(4) After redeploying the code, your changes should now be a part of the default Metron dashboard. - -#### Why do my dashboard components change their order when reloading the dashboard? - -This has been a problem in Kibana 4.5.1 and perhaps other versions too. To address this problem find the definition for your dashboard in the Kibana index extract. It will look like the following. - -``` -{"_index":".kibana","_type":"dashboard","_id":"Metron-Dashboard",... -``` - -Extract the `panelsJSON` field from the dashboard definition. Reorder the definition of these panels so that they are ordered by row and column. The component in row 1 should come before the component in row 2, etc. After you have ordered the components in this way, Kibana will maintain the order of components in the dashboard. diff --git a/metron-deployment/roles/kibana/defaults/main.yml b/metron-deployment/roles/kibana/defaults/main.yml deleted file mode 100644 index c7e291b024..0000000000 --- a/metron-deployment/roles/kibana/defaults/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -kibana_repo_url: http://packages.elastic.co/kibana/4.5/centos -kibana_repo_key_url: http://packages.elastic.co/GPG-KEY-elasticsearch -kibana_index_url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/.kibana" -kibana_index_def: "/tmp/kibana-index.json" -kibana_version: 4.5.3 -nodesource_repo_setup: https://rpm.nodesource.com/setup_4.x diff --git a/metron-deployment/roles/kibana/tasks/dashboard.yml b/metron-deployment/roles/kibana/tasks/dashboard.yml deleted file mode 100644 index 7bfadde636..0000000000 --- a/metron-deployment/roles/kibana/tasks/dashboard.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Distribute the Kibana index containing the Metron Dashboard - template: src=kibana-index.json dest={{ kibana_index_def }} - -- name: Install the Metron Dashboard - shell: "elasticdump --output={{ kibana_index_url }} --input={{ kibana_index_def }}" diff --git a/metron-deployment/roles/kibana/tasks/dependencies.yml b/metron-deployment/roles/kibana/tasks/dependencies.yml deleted file mode 100644 index 7450665b9b..0000000000 --- a/metron-deployment/roles/kibana/tasks/dependencies.yml +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Load Elastic Public Signing Key - rpm_key: - state: present - key: "{{ kibana_repo_key_url }}" - -- name: Create kibana.repo - template: - src: kibana.repo.template - dest: /etc/yum.repos.d/kibana.repo - mode: 0644 diff --git a/metron-deployment/roles/kibana/tasks/elasticdump.yml b/metron-deployment/roles/kibana/tasks/elasticdump.yml deleted file mode 100644 index 739936ff53..0000000000 --- a/metron-deployment/roles/kibana/tasks/elasticdump.yml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Download Nodesource Yum Repository Setup - shell: - cmd: curl -s {{ nodesource_repo_setup }} -o /tmp/nodesource_setup_4.x - creates: /tmp/nodesource_setup_4.x - warn: false - -- name: Setup Nodesource Yum Repository - shell: bash /tmp/nodesource_setup_4.x - -- name: Install Node - Needed for Elasticdump - yum: name="{{ item }}" - with_items: - - gcc-c++ - - make - - nodejs - -- name: Install Elasticdump - shell: npm install elasticdump -g diff --git a/metron-deployment/roles/kibana/tasks/kibana.yml b/metron-deployment/roles/kibana/tasks/kibana.yml deleted file mode 100644 index 524437c9b9..0000000000 --- a/metron-deployment/roles/kibana/tasks/kibana.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Install Kibana - yum: - name: "kibana-{{ kibana_version }}" - state: installed - register: result - until: result.rc == 0 - retries: 5 - delay: 10 - -- name: Configure Kibana - remove stale entries - lineinfile: > - dest=/opt/kibana/config/kibana.yml - regexp="{{ item }}" - state=absent - with_items: - - '^server\.port:' - - '^elasticsearch\.url:' - - '^default_app_id:' - -- name: Configure Kibana - lineinfile: > - dest=/opt/kibana/config/kibana.yml - regexp="{{ item.regexp }}" - insertafter="{{ item.regexp }}" - line="{{ item.line }}" - state=present - with_items: - - { regexp: '^# *server\.port:', line: 'server.port: 5000' } - - { regexp: '^# *elasticsearch\.url:', line: 'elasticsearch.url: http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}' } - - { regexp: '^# *default_app_id:', line: 'default_app_id: "dashboard/Metron-Dashboard"' } diff --git a/metron-deployment/roles/kibana/tasks/main.yml b/metron-deployment/roles/kibana/tasks/main.yml deleted file mode 100644 index f125e33bc9..0000000000 --- a/metron-deployment/roles/kibana/tasks/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- include: dependencies.yml -- include: kibana.yml -- include: elasticdump.yml -- include: dashboard.yml diff --git a/metron-deployment/roles/kibana/templates/kibana-index.json b/metron-deployment/roles/kibana/templates/kibana-index.json deleted file mode 100644 index 7dcf04f571..0000000000 --- a/metron-deployment/roles/kibana/templates/kibana-index.json +++ /dev/null @@ -1,34 +0,0 @@ -{"_index":".kibana","_type":"index-pattern","_id":"bro*","_score":1,"_source":{"title":"bro*","timeFieldName":"timestamp","fields":"[{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"qclass_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bro_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"answers\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":1,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"resp_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",\"count\":1,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"method\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rcode\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"orig_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"orig_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"proto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"qtype_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"status_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rejected\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"qtype\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"trans_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"uid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"trans_depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"qclass\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"resp_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"request_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rcode_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"referrer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"status_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":2,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]"}} -{"_index":".kibana","_type":"index-pattern","_id":"yaf*","_score":1,"_source":{"title":"yaf*","timeFieldName":"timestamp","fields":"[{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"isn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"rpkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"oct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"end_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"riflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"proto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"iflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"uflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ruflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"roct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"sip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"sp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rtag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"end-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"risn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"end_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rtt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]"}} -{"_index":".kibana","_type":"index-pattern","_id":"snort*","_score":1,"_source":{"title":"snort*","timeFieldName":"timestamp","fields":"[{\"name\":\"msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dgmlen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tcpack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ethlen\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threat:triage:level\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tcpflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"sig_rev\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"ethsrc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tcpseq\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tcpwindow\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tos\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ethdst\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"is_alert\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ttl\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"iplen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"sig_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"sig_generator\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]"}} -{"_index":".kibana","_type":"config","_id":"{{ kibana_version }}","_score":1,"_source":{"buildNum":9892,"defaultIndex":"bro*"}} -{"_index":".kibana","_type":"search","_id":"web-search","_score":1,"_source":{"title":"Web Requests","description":"","hits":0,"columns":["method","host","uri","referrer","ip_src_addr","ip_dst_addr"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"bro*\",\"query\":{\"query_string\":{\"query\":\"protocol: http OR protocol: https\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"}}} -{"_index":".kibana","_type":"search","_id":"yaf-search","_score":1,"_source":{"title":"YAF","description":"","hits":0,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"yaf*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}"}}} -{"_index":".kibana","_type":"search","_id":"snort-search","_score":1,"_source":{"title":"Snort Alerts","description":"","hits":0,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"snort*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"}}} -{"_index":".kibana","_type":"search","_id":"dns-search","_score":1,"_source":{"title":"DNS Requests","description":"","hits":0,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"bro*\",\"query\":{\"query_string\":{\"query\":\"protocol: dns\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"}}} -{"_index":".kibana","_type":"visualization","_id":"Welcome","_score":1,"_source":{"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Top-Snort-Alerts-by-Source","_score":1,"_source":{"title":"Top Snort Alerts by Source","visState":"{\"title\":\"Top Snort Alerts by Source\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_src_addr\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source IP\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"snort*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Web-Request-Type","_score":1,"_source":{"title":"Web Request Type","visState":"{\"title\":\"Web Request Type\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"method\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","savedSearchId":"web-search","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Web-Request-Header","_score":1,"_source":{"title":"Web Request Header","visState":"{\"title\":\"Web Request Header\",\"type\":\"markdown\",\"params\":{\"markdown\":\"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. \"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Events","_score":1,"_source":{"title":"Events","visState":"{\"title\":\"Events\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"source:type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":[\"yaf*\", \"bro*\", \"snort*\"],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Flow-Duration","_score":1,"_source":{"title":"Flow Duration","visState":"{\"title\":\"Flow Duration\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":false,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"duration\",\"interval\":10,\"extended_bounds\":{},\"customLabel\":\"Flow Duration (seconds)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"yaf*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Snort-Header","_score":1,"_source":{"title":"Snort","visState":"{\"title\":\"Snort\",\"type\":\"markdown\",\"params\":{\"markdown\":\"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Unique-Location(s)","_score":1,"_source":{"title":"Geo-IP Locations","visState":"{\"title\":\"Geo-IP Locations\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:locID\",\"customLabel\":\"Unique Location(s)\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":[\"yaf*\", \"bro*\", \"snort*\"],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Event-Types","_score":1,"_source":{"title":"Event Sources","visState":"{\"title\":\"Event Sources\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source:type\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":[\"yaf*\", \"bro*\", \"snort*\"],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Total-Events","_score":1,"_source":{"title":"Event Count","visState":"{\"title\":\"Event Count\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Events\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":[\"yaf*\", \"bro*\", \"snort*\"],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Top-DNS-Query","_score":1,"_source":{"title":"Top DNS Query","visState":"{\"title\":\"Top DNS Query\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"query\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"bro*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"YAF-Flow(s)","_score":1,"_source":{"title":"YAF Flows","visState":"{\"title\":\"YAF Flows\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"yaf*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Top-Alerts-By-Host","_score":1,"_source":{"title":"Top Alerts By Host","visState":"{\"title\":\"New Visualization\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_src_addr\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_dst_addr\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Destination\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","savedSearchId":"snort-search","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Location-Header","_score":1,"_source":{"title":"Enrichment","visState":"{\"title\":\"Enrichment\",\"type\":\"markdown\",\"params\":{\"markdown\":\"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Frequent-DNS-Queries","_score":1,"_source":{"title":"Frequent DNS Requests","visState":"{\"title\":\"Frequent DNS Requests\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"query\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"bro*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Snort-Alert-Types","_score":1,"_source":{"title":"Snort Alert Types","visState":"{\"title\":\"Snort Alert Types\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sig_id\",\"customLabel\":\"Alert Type(s)\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"snort*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"HTTP(S)-Requests","_score":1,"_source":{"title":"Web Requests","visState":"{\"title\":\"Web Requests\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","savedSearchId":"web-search","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"DNS-Request(s)","_score":1,"_source":{"title":"DNS Requests","visState":"{\"title\":\"DNS Requests\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","savedSearchId":"dns-search","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"YAF-Flows-Header","_score":1,"_source":{"title":"YAF","visState":"{\"title\":\"YAF\",\"type\":\"markdown\",\"params\":{\"markdown\":\"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"DNS-Requests-Header","_score":1,"_source":{"title":"DNS Requests","visState":"{\"aggs\":[],\"listeners\":{},\"params\":{\"markdown\":\"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network.\"},\"title\":\"DNS Requests\",\"type\":\"markdown\"}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Top-Destinations","_score":1,"_source":{"title":"Top Destinations","visState":"{\"title\":\"Top Destinations\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_dst_addr\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Destination IP\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":[\"yaf*\", \"bro*\", \"snort*\"],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Country","_score":1,"_source":{"title":"By Country","visState":"{\"title\":\"By Country\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:country\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":[\"yaf*\", \"bro*\", \"snort*\"],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Frequent-DNS-Requests","_score":1,"_source":{"title":"Frequent DNS Requests","visState":"{\"title\":\"Frequent DNS Requests\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"query\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"DNS Query\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"bro*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Unusual-Referrers","_score":1,"_source":{"title":"Unusual Referrers","visState":"{\"title\":\"Unusual Referrers\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"significant_terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"referrer\",\"size\":5,\"customLabel\":\"Top 5 Unusual Referrers\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","savedSearchId":"web-search","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"}}} -{"_index":".kibana","_type":"visualization","_id":"Flow-Locations","_score":1,"_source":{"title":"Flow Locations","visState":"{\"title\":\"New Visualization\",\"type\":\"tile_map\",\"params\":{\"mapType\":\"Scaled Circle Markers\",\"isDesaturated\":true,\"addTooltip\":true,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatRadius\":25,\"heatBlur\":15,\"heatNormalizeData\":true,\"wms\":{\"enabled\":false,\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\"options\":{\"version\":\"1.3.0\",\"layers\":\"0\",\"format\":\"image/png\",\"transparent\":true,\"attribution\":\"Maps provided by USGS\",\"styles\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"enrichments:geo:ip_dst_addr:location_point\",\"autoPrecision\":true,\"precision\":2}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":[\"yaf*\", \"bro*\", \"snort*\"],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}} -{"_index":".kibana","_type":"dashboard","_id":"Metron-Dashboard","_score":1,"_source":{"title":"Metron Dashboard","hits":0,"description":"","panelsJSON":"[{\"col\":1,\"id\":\"Welcome\",\"panelIndex\":30,\"row\":1,\"size_x\":11,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Total-Events\",\"panelIndex\":6,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Events\",\"panelIndex\":16,\"row\":3,\"size_x\":8,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Event-Types\",\"panelIndex\":15,\"row\":5,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Location-Header\",\"panelIndex\":24,\"row\":7,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Unique-Location(s)\",\"panelIndex\":23,\"row\":9,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Flow-Locations\",\"panelIndex\":32,\"row\":7,\"size_x\":8,\"size_y\":6,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Country\",\"panelIndex\":8,\"row\":11,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"YAF-Flows-Header\",\"panelIndex\":27,\"row\":13,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"YAF-Flow(s)\",\"panelIndex\":21,\"row\":15,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"ip_src_addr\",\"ip_src_port\",\"ip_dst_addr\",\"ip_dst_port\",\"protocol\",\"duration\",\"pkt\"],\"id\":\"yaf-search\",\"panelIndex\":20,\"row\":13,\"size_x\":8,\"size_y\":6,\"sort\":[\"duration\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Flow-Duration\",\"panelIndex\":31,\"row\":17,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Snort-Header\",\"panelIndex\":25,\"row\":19,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"msg\",\"sig_id\",\"ip_src_addr\",\"ip_src_port\",\"ip_dst_addr\",\"ip_dst_port\"],\"id\":\"snort-search\",\"panelIndex\":3,\"row\":19,\"size_x\":8,\"size_y\":6,\"sort\":[\"timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Snort-Alert-Types\",\"panelIndex\":10,\"row\":21,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-Alerts-By-Host\",\"panelIndex\":19,\"row\":23,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Web-Request-Header\",\"panelIndex\":26,\"row\":25,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"method\",\"host\",\"uri\",\"referrer\",\"user_agent\",\"ip_src_addr\",\"ip_dst_addr\"],\"id\":\"web-search\",\"panelIndex\":4,\"row\":25,\"size_x\":8,\"size_y\":6,\"sort\":[\"timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"HTTP(S)-Requests\",\"panelIndex\":17,\"row\":27,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"DNS-Requests-Header\",\"panelIndex\":29,\"row\":31,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"query\",\"qtype_name\",\"answers\",\"ip_src_addr\",\"ip_dst_addr\"],\"id\":\"dns-search\",\"panelIndex\":5,\"row\":31,\"size_x\":8,\"size_y\":6,\"sort\":[\"timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"DNS-Request(s)\",\"panelIndex\":14,\"row\":33,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Web-Request-Type\",\"panelIndex\":33,\"row\":29,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"}]","optionsJSON":"{\"darkTheme\":false}","uiStateJSON":"{\"P-23\":{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}},\"P-34\":{\"vis\":{\"legendOpen\":false}}}","version":1,"timeRestore":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"}}} diff --git a/metron-deployment/roles/kibana/templates/kibana.repo.template b/metron-deployment/roles/kibana/templates/kibana.repo.template deleted file mode 100644 index d5c096fd97..0000000000 --- a/metron-deployment/roles/kibana/templates/kibana.repo.template +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -[kibana] -name=Kibana installation repository -baseurl={{ kibana_repo_url }} -gpgcheck=1 -gpgkey={{ kibana_repo_key_url }} -enabled=1 diff --git a/metron-deployment/roles/metron_hbase_tables/meta/main.yml b/metron-deployment/roles/load_web_templates/meta/main.yml similarity index 100% rename from metron-deployment/roles/metron_hbase_tables/meta/main.yml rename to metron-deployment/roles/load_web_templates/meta/main.yml diff --git a/metron-deployment/roles/load_web_templates/tasks/main.yml b/metron-deployment/roles/load_web_templates/tasks/main.yml new file mode 100644 index 0000000000..a5dbbbaf5b --- /dev/null +++ b/metron-deployment/roles/load_web_templates/tasks/main.yml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Load ES Templates + command: > + curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install ES Template from REST", "command": "ELASTICSEARCH_TEMPLATE_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "{{ metron_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests + args: + warn: off + register: result + failed_when: "result.rc != 0 or '202' not in result.stdout" + +- name: Load Kibana Dashboard + command: > + curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "{{ kibana_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests + args: + warn: off + register: result + failed_when: "result.rc != 0 or '202' not in result.stdout" diff --git a/metron-deployment/roles/elasticsearch/tasks/main.yml b/metron-deployment/roles/metron-rpms/tasks/main.yml similarity index 72% rename from metron-deployment/roles/elasticsearch/tasks/main.yml rename to metron-deployment/roles/metron-rpms/tasks/main.yml index 7478842c13..c22487cce6 100644 --- a/metron-deployment/roles/elasticsearch/tasks/main.yml +++ b/metron-deployment/roles/metron-rpms/tasks/main.yml @@ -15,10 +15,18 @@ # limitations under the License. # --- -- include: elasticsearch.yml -- name: Create Logrotate Script for Elasticsearch - template: - src: "metron-elasticsearch-logrotate.yml" - dest: "/etc/logrotate.d/metron-elasticsearch" - mode: 0644 +- name: Create localrepo directory + file: + path: /localrepo + state: directory + mode: 0755 + +- name: Copy Metron RPMs + copy: + src: "{{ item }}" + dest: /localrepo + owner: root + mode: 0755 + with_fileglob: + - "{{ playbook_dir }}/../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm" \ No newline at end of file diff --git a/metron-deployment/roles/metron_common/defaults/main.yml b/metron-deployment/roles/metron_common/defaults/main.yml deleted file mode 100644 index e4a773576c..0000000000 --- a/metron-deployment/roles/metron_common/defaults/main.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -metron_jar_name: metron-elasticsearch-{{ metron_version }}.jar -metron_jar_path: "{{ playbook_dir }}/../../metron-platform/metron-elasticsearch/target/{{ metron_jar_name }}" diff --git a/metron-deployment/roles/metron_common/tasks/main.yml b/metron-deployment/roles/metron_common/tasks/main.yml deleted file mode 100644 index f528b31804..0000000000 --- a/metron-deployment/roles/metron_common/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Check OS Version - fail: msg="Metron deployment supports CentOS 6 only." - when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6") - -- name: Check for Metron jar path - become: false - local_action: stat path={{ metron_jar_path }} - register: metron_jars - -- name: Verify Metron jars exist - fail: msg="Unable to locate staged Metron jars at {{ metron_jar_path }}. Did you run 'mvn package'?" - when: not metron_jars.stat.exists - -- name: Ensure iptables is stopped and is not running at boot time. - ignore_errors: yes - service: - name: iptables - state: stopped - enabled: no diff --git a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/bro_index.template b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/bro_index.template deleted file mode 100644 index 8045c95167..0000000000 --- a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/bro_index.template +++ /dev/null @@ -1,218 +0,0 @@ -{ - "template": "bro_index*", - "mappings": { - "bro_doc": { - "_timestamp": { - "enabled": true - }, - "dynamic_templates": [ - { - "geo_location_point": { - "match": "enrichments:geo:*:location_point", - "match_mapping_type": "*", - "mapping": { - "type": "geo_point" - } - } - }, - { - "geo_country": { - "match": "enrichments:geo:*:country", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_city": { - "match": "enrichments:geo:*:city", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_location_id": { - "match": "enrichments:geo:*:locID", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_dma_code": { - "match": "enrichments:geo:*:dmaCode", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_postal_code": { - "match": "enrichments:geo:*:postalCode", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_latitude": { - "match": "enrichments:geo:*:latitude", - "match_mapping_type": "*", - "mapping": { - "type": "float" - } - } - }, - { - "geo_longitude": { - "match": "enrichments:geo:*:longitude", - "match_mapping_type": "*", - "mapping": { - "type": "float" - } - } - }, - { - "timestamps": { - "match": "*:ts", - "match_mapping_type": "*", - "mapping": { - "type": "date", - "format": "epoch_millis" - } - } - } - ], - "properties": { - "timestamp": { - "type": "date", - "format": "epoch_millis" - }, - "source:type": { - "type": "string", - "index": "not_analyzed" - }, - "ip_dst_addr": { - "type": "ip" - }, - "ip_dst_port": { - "type": "integer" - }, - "ip_src_addr": { - "type": "ip" - }, - "ip_src_port": { - "type": "integer" - }, - "status_code": { - "type": "integer" - }, - "method": { - "type": "string", - "index": "not_analyzed" - }, - "protocol": { - "type": "string", - "index": "not_analyzed" - }, - "request_body_len": { - "type": "long" - }, - "uri": { - "type": "string", - "index": "not_analyzed", - "ignore_above": 8191 - }, - "uid": { - "type": "string", - "index": "not_analyzed" - }, - "referrer": { - "type": "string", - "index": "not_analyzed" - }, - "trans_depth": { - "type": "integer" - }, - "host": { - "type": "string", - "index": "not_analyzed" - }, - "status_msg": { - "type": "string", - "index": "not_analyzed" - }, - "response_body_len": { - "type": "long" - }, - "user_agent": { - "type": "string" - }, - "query": { - "type": "string", - "index": "not_analyzed" - }, - "answers": { - "type": "string" - }, - "AA": { - "type": "boolean" - }, - "TC": { - "type": "boolean" - }, - "RA": { - "type": "boolean" - }, - "RD": { - "type": "boolean" - }, - "rejected": { - "type": "boolean" - }, - "qclass_name": { - "type": "string", - "index": "not_analyzed" - }, - "proto": { - "type": "string", - "index": "not_analyzed" - }, - "rcode": { - "type": "integer" - }, - "rcode_name": { - "type": "string", - "index": "not_analyzed" - }, - "trans_id": { - "type": "integer" - }, - "Z": { - "type": "integer" - }, - "qclass": { - "type": "integer" - }, - "qtype": { - "type": "integer" - }, - "qtype_name": { - "type": "string", - "index": "not_analyzed" - } - } - } - } -} diff --git a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/snort_index.template b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/snort_index.template deleted file mode 100644 index bf943dfc95..0000000000 --- a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/snort_index.template +++ /dev/null @@ -1,183 +0,0 @@ -{ - "template": "snort_index*", - "mappings": { - "snort_doc": { - "_timestamp": { - "enabled": true - }, - "dynamic_templates": [ - { - "geo_location_point": { - "match": "enrichments:geo:*:location_point", - "match_mapping_type": "*", - "mapping": { - "type": "geo_point" - } - } - }, - { - "geo_country": { - "match": "enrichments:geo:*:country", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_city": { - "match": "enrichments:geo:*:city", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_location_id": { - "match": "enrichments:geo:*:locID", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_dma_code": { - "match": "enrichments:geo:*:dmaCode", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_postal_code": { - "match": "enrichments:geo:*:postalCode", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_latitude": { - "match": "enrichments:geo:*:latitude", - "match_mapping_type": "*", - "mapping": { - "type": "float" - } - } - }, - { - "geo_longitude": { - "match": "enrichments:geo:*:longitude", - "match_mapping_type": "*", - "mapping": { - "type": "float" - } - } - }, - { - "timestamps": { - "match": "*:ts", - "match_mapping_type": "*", - "mapping": { - "type": "date", - "format": "epoch_millis" - } - } - } - ], - "properties": { - "timestamp": { - "type": "date", - "format": "epoch_millis" - }, - "source:type": { - "type": "string", - "index": "not_analyzed" - }, - "ip_dst_addr": { - "type": "ip" - }, - "ip_dst_port": { - "type": "integer" - }, - "ip_src_addr": { - "type": "ip" - }, - "ip_src_port": { - "type": "integer" - }, - "dgmlen": { - "type": "integer" - }, - "ethdst": { - "type": "string", - "index": "not_analyzed" - }, - "ethlen": { - "type": "string", - "index": "not_analyzed" - }, - "ethsrc": { - "type": "string", - "index": "not_analyzed" - }, - "id": { - "type": "integer" - }, - "iplen": { - "type": "integer" - }, - "is_alert": { - "type": "boolean" - }, - "msg": { - "type": "string" - }, - "protocol": { - "type": "string", - "index": "not_analyzed" - }, - "sig_generator": { - "type": "string", - "index": "not_analyzed" - }, - "sig_id": { - "type": "integer" - }, - "sig_rev": { - "type": "string" - }, - "tcpack": { - "type": "string" - }, - "tcpflags": { - "type": "string" - }, - "tcpseq": { - "type": "string" - }, - "tcpwindow": { - "type": "string" - }, - "threat:triage:level": { - "type": "double" - }, - "tos": { - "type": "integer" - }, - "ttl": { - "type": "integer" - } - } - } - } -} diff --git a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template deleted file mode 100644 index 7743afcd78..0000000000 --- a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template +++ /dev/null @@ -1,205 +0,0 @@ -{ - "template": "yaf_index*", - "mappings": { - "yaf_doc": { - "_timestamp": { - "enabled": true - }, - "dynamic_templates": [ - { - "geo_location_point": { - "match": "enrichments:geo:*:location_point", - "match_mapping_type": "*", - "mapping": { - "type": "geo_point" - } - } - }, - { - "geo_country": { - "match": "enrichments:geo:*:country", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_city": { - "match": "enrichments:geo:*:city", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_location_id": { - "match": "enrichments:geo:*:locID", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_dma_code": { - "match": "enrichments:geo:*:dmaCode", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_postal_code": { - "match": "enrichments:geo:*:postalCode", - "match_mapping_type": "*", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - }, - { - "geo_latitude": { - "match": "enrichments:geo:*:latitude", - "match_mapping_type": "*", - "mapping": { - "type": "float" - } - } - }, - { - "geo_longitude": { - "match": "enrichments:geo:*:longitude", - "match_mapping_type": "*", - "mapping": { - "type": "float" - } - } - }, - { - "timestamps": { - "match": "*:ts", - "match_mapping_type": "*", - "mapping": { - "type": "date", - "format": "epoch_millis" - } - } - } - ], - "properties": { - "timestamp": { - "type": "date", - "format": "epoch_millis" - }, - "source:type": { - "type": "string", - "index": "not_analyzed" - }, - "ip_dst_addr": { - "type": "ip" - }, - "ip_dst_port": { - "type": "integer" - }, - "ip_src_addr": { - "type": "ip" - }, - "ip_src_port": { - "type": "integer" - }, - "start_time": { - "type": "date", - "format": "epoch_millis" - }, - "end_time": { - "type": "date", - "format": "epoch_millis" - }, - "duration": { - "type": "double" - }, - "rtt": { - "type": "double" - }, - "proto": { - "type": "string", - "index": "not_analyzed" - }, - "sip": { - "type": "string", - "index": "not_analyzed" - }, - "sp": { - "type": "string", - "index": "not_analyzed" - }, - "dip": { - "type": "string", - "index": "not_analyzed" - }, - "dp": { - "type": "string", - "index": "not_analyzed" - }, - "iflags": { - "type": "string", - "index": "not_analyzed" - }, - "uflags": { - "type": "string", - "index": "not_analyzed" - }, - "riflags": { - "type": "string" - }, - "ruflags": { - "type": "string", - "index": "not_analyzed" - }, - "isn": { - "type": "string", - "index": "not_analyzed" - }, - "risn": { - "type": "string", - "index": "not_analyzed" - }, - "tag": { - "type": "string", - "index": "not_analyzed" - }, - "rtag": { - "type": "string", - "index": "not_analyzed" - }, - "pkt": { - "type": "integer" - }, - "oct": { - "type": "integer" - }, - "rpkt": { - "type": "integer" - }, - "roct": { - "type": "integer" - }, - "app": { - "type": "string", - "index": "not_analyzed" - }, - "end-reason": { - "type": "string" - } - } - } - } -} diff --git a/metron-deployment/roles/metron_elasticsearch_templates/tasks/load_templates.yml b/metron-deployment/roles/metron_elasticsearch_templates/tasks/load_templates.yml deleted file mode 100644 index 812569db54..0000000000 --- a/metron-deployment/roles/metron_elasticsearch_templates/tasks/load_templates.yml +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Start Elasticsearch - service: name=elasticsearch state=started - -- name : Wait for Elasticsearch Host to Start - wait_for: - host: "{{ inventory_hostname }}" - port: "{{ elasticsearch_web_port }}" - delay: 10 - timeout: 300 - -- name: Wait for Index to Become Available - uri: - url: "http://{{ inventory_hostname }}:{{ elasticsearch_web_port }}/_cat/health" - method: GET - status_code: 200 - return_content: yes - register: result - until: result.content.find("green") != -1 or result.content.find("yellow") != -1 - retries: 10 - delay: 60 - -- name: Add Elasticsearch templates for topologies - uri: - url: "http://{{ inventory_hostname }}:{{ elasticsearch_web_port }}/_template/{{ item | basename | replace('.template','') }}" - method: PUT - body: "{{ lookup('file',item) }}" - status_code: 200 - with_fileglob: ./files/es_templates/*.template - -- name: Validate Elasticsearch templates - uri: - url: "http://{{ inventory_hostname }}:{{ elasticsearch_web_port }}/_template/{{ item | basename | replace('.template','') }}" - method: HEAD - body: "{{ lookup('file',item) }}" - status_code: 200 - with_fileglob: ./files/es_templates/*.template diff --git a/metron-deployment/roles/metron_elasticsearch_templates/tasks/main.yml b/metron-deployment/roles/metron_elasticsearch_templates/tasks/main.yml deleted file mode 100644 index 61fb625891..0000000000 --- a/metron-deployment/roles/metron_elasticsearch_templates/tasks/main.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- include: load_templates.yml diff --git a/metron-deployment/roles/metron_hbase_tables/defaults/main.yml b/metron-deployment/roles/metron_hbase_tables/defaults/main.yml deleted file mode 100644 index 9d40d1b9c7..0000000000 --- a/metron-deployment/roles/metron_hbase_tables/defaults/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -hbase_tables_to_create: - - pcap - - access_tracker - - threatintel - - enrichment diff --git a/metron-deployment/roles/metron_hbase_tables/tasks/main.yml b/metron-deployment/roles/metron_hbase_tables/tasks/main.yml deleted file mode 100644 index cc86c09652..0000000000 --- a/metron-deployment/roles/metron_hbase_tables/tasks/main.yml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# must run on hadoop host -- name: Create the HBase tables required for Metron - shell: echo "create '{{ item }}','t'" | hbase shell -n - with_items: "{{ hbase_tables_to_create }}" - register: out - failed_when: out.rc != 0 and ("Table already exists" not in out.stdout) - changed_when: ("Table already exists" not in out.stdout) diff --git a/metron-deployment/roles/metron_kafka_topics/defaults/main.yml b/metron-deployment/roles/metron_kafka_topics/defaults/main.yml deleted file mode 100644 index 4a97a8e509..0000000000 --- a/metron-deployment/roles/metron_kafka_topics/defaults/main.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -kafka_home: /usr/hdp/current/kafka-broker/ -topics_to_create: - - { topic: "pcap", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "bro", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "yaf", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "snort", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "enrichments", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "enrichments_error", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "threatintel_error", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "parser_invalid", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "parser_error", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "indexing", num_partitions: 1, replication_factor: 1, retention_gb: 10 } - - { topic: "indexing_error", num_partitions: 1, replication_factor: 1, retention_gb: 10 } diff --git a/metron-deployment/roles/metron_kafka_topics/meta/main.yml b/metron-deployment/roles/metron_kafka_topics/meta/main.yml deleted file mode 100644 index c3d807b119..0000000000 --- a/metron-deployment/roles/metron_kafka_topics/meta/main.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -dependencies: - - ambari_gather_facts diff --git a/metron-deployment/roles/metron_kafka_topics/tasks/main.yml b/metron-deployment/roles/metron_kafka_topics/tasks/main.yml deleted file mode 100644 index 8d3cb0a05f..0000000000 --- a/metron-deployment/roles/metron_kafka_topics/tasks/main.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create the Kafka topics required for Metron - shell: > - {{ kafka_home }}/bin/kafka-topics.sh \ - --zookeeper {{ zookeeper_url }} \ - --create \ - --topic {{ item.topic }} \ - --partitions {{ item.num_partitions }} \ - --replication-factor {{ item.replication_factor }} \ - --config retention.bytes={{ item.retention_gb * 1024 * 1024 * 1024 }} - with_items: "{{ topics_to_create }}" - register: out - failed_when: out.rc != 0 and ("already exists" not in out.stderr) - changed_when: ("already exists" not in out.stderr) diff --git a/metron-deployment/roles/monit/defaults/main.yml b/metron-deployment/roles/monit/defaults/main.yml index c1c7583b30..651aa58170 100644 --- a/metron-deployment/roles/monit/defaults/main.yml +++ b/metron-deployment/roles/monit/defaults/main.yml @@ -19,9 +19,6 @@ monit_home: /usr/local/monit monit_config_home: /etc/monit.d monit_user: admin monit_pass: monit -topology_start_timeout: 120 -topology_stop_timeout: 120 bro_pid_file: /usr/local/bro/spool/bro/.pid -elasticsearch_pid_file: /var/run/elasticsearch/elasticsearch.pid snort_alert_csv_path: /var/log/snort/alert.csv diff --git a/metron-deployment/roles/monit/tasks/main.yml b/metron-deployment/roles/monit/tasks/main.yml index 3718797b10..9c1d75d2f1 100644 --- a/metron-deployment/roles/monit/tasks/main.yml +++ b/metron-deployment/roles/monit/tasks/main.yml @@ -17,7 +17,6 @@ # --- - include: monit.yml -- include: scripts.yml - include: monit-definitions.yml - include: monit-sensor-definitions.yml @@ -26,4 +25,4 @@ - include: monit-stub-definitions.yml tags: - - sensor-stubs \ No newline at end of file + - sensor-stubs diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml index b6ea82194c..c9f5f0788e 100644 --- a/metron-deployment/roles/monit/tasks/monit-definitions.yml +++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml @@ -16,26 +16,6 @@ # limitations under the License. # --- -- name: Create monit definition for elasticsearch - template: src=monit/elasticsearch.monit dest={{ monit_config_home }}/elasticsearch.monit - when: ("search" in group_names) and (install_elasticsearch | default(True)) - -- name: Create monit definition for indexing with elasticsearch - template: src=monit/indexing-elasticsearch.monit dest={{ monit_config_home }}/indexing-elasticsearch.monit - when: ("enrichment" in group_names) and (install_elasticsearch | default(True)) - -- name: Create monit definition for enrichment - template: src=monit/enrichment.monit dest={{ monit_config_home }}/enrichment.monit - when: ("enrichment" in group_names) - -- name: Create monit definition for kibana - template: src=monit/kibana.monit dest={{ monit_config_home }}/kibana.monit - when: ("web" in group_names) and (install_elasticsearch | default(True)) - -- name: Create monit definition for parsers - template: src=monit/parsers.monit dest={{ monit_config_home }}/parsers.monit - when: ("enrichment" in group_names) - - name: Create monit definition for pcap-replay template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit when: ("sensors" in group_names) and (install_pcap_replay | default(False)) diff --git a/metron-deployment/roles/monit/tasks/scripts.yml b/metron-deployment/roles/monit/tasks/scripts.yml deleted file mode 100644 index 43901d214e..0000000000 --- a/metron-deployment/roles/monit/tasks/scripts.yml +++ /dev/null @@ -1,55 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create monit home - file: path={{ monit_home }} state=directory mode=0755 - -- name: Deploy parser topology start scripts - template: - src: scripts/start_topology.sh - dest: "{{ monit_home }}/start_{{ item }}_topology.sh" - mode: 0755 - with_items: - - yaf - - snort - - bro - -- name: Deploy topology stop scripts - template: - src: scripts/stop_topology.sh - dest: "{{ monit_home }}/stop_{{ item }}_topology.sh" - mode: 0755 - with_items: - - yaf - - snort - - bro - - pcap - - enrichment - - indexing - -- name: Deploy topology status scripts - template: - src: scripts/status_topology.sh - dest: "{{ monit_home }}/status_{{ item }}_topology.sh" - mode: 0755 - with_items: - - yaf - - snort - - bro - - pcap - - enrichment - - indexing diff --git a/metron-deployment/roles/monit/templates/monit/elasticsearch.monit b/metron-deployment/roles/monit/templates/monit/elasticsearch.monit deleted file mode 100644 index 805c3cb98c..0000000000 --- a/metron-deployment/roles/monit/templates/monit/elasticsearch.monit +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check process elasticsearch with pidfile {{ elasticsearch_pid_file }} - start program = "/etc/init.d/elasticsearch start" - stop program = "/etc/init.d/elasticsearch stop" - if does not exist then restart - group enrichments - group search - group metron diff --git a/metron-deployment/roles/monit/templates/monit/enrichment.monit b/metron-deployment/roles/monit/templates/monit/enrichment.monit deleted file mode 100644 index c1c61d1728..0000000000 --- a/metron-deployment/roles/monit/templates/monit/enrichment.monit +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh" - start program "{{ metron_directory }}/bin/start_enrichment_topology.sh" with timeout {{ topology_start_timeout }} seconds - stop program "{{ monit_home }}/stop_enrichment_topology.sh" with timeout {{ topology_stop_timeout }} seconds - if status != 0 then restart - group yaf - group bro - group snort - group enrichments - group metron diff --git a/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit b/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit deleted file mode 100644 index af60f44234..0000000000 --- a/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check program indexing with path "{{ monit_home }}/status_indexing_topology.sh" - start program "{{ metron_directory }}/bin/start_elasticsearch_topology.sh" with timeout {{ topology_start_timeout }} seconds - stop program "{{ monit_home }}/stop_indexing_topology.sh" with timeout {{ topology_stop_timeout }} seconds - if status != 0 then restart - group yaf - group bro - group snort - group enrichments - group metron diff --git a/metron-deployment/roles/monit/templates/monit/indexing-solr.monit b/metron-deployment/roles/monit/templates/monit/indexing-solr.monit deleted file mode 100644 index 6c7a508cc0..0000000000 --- a/metron-deployment/roles/monit/templates/monit/indexing-solr.monit +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check program indexing with path "{{ monit_home }}/status_indexing_topology.sh" - start program "{{ metron_directory }}/bin/start_solr_topology.sh" with timeout {{ topology_start_timeout }} seconds - stop program "{{ monit_home }}/stop_indexing_topology.sh" with timeout {{ topology_stop_timeout }} seconds - if status != 0 then restart - group yaf - group bro - group snort - group enrichments - group metron diff --git a/metron-deployment/roles/monit/templates/monit/kibana.monit b/metron-deployment/roles/monit/templates/monit/kibana.monit deleted file mode 100644 index 41b4cb9492..0000000000 --- a/metron-deployment/roles/monit/templates/monit/kibana.monit +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check process kibana matching "/opt/kibana/bin/../node/bin/node /opt/kibana/bin/../src/cli" - start program = "/etc/init.d/kibana start" - stop program = "/etc/init.d/kibana stop" - if does not exist then restart - group web - group metron diff --git a/metron-deployment/roles/monit/templates/monit/parsers.monit b/metron-deployment/roles/monit/templates/monit/parsers.monit deleted file mode 100644 index 1eff8b601a..0000000000 --- a/metron-deployment/roles/monit/templates/monit/parsers.monit +++ /dev/null @@ -1,49 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -check program pcap-parser with path "{{ monit_home }}/status_pcap_topology.sh" - start program "{{ metron_directory }}/bin/start_pcap_topology.sh" with timeout {{ topology_start_timeout }} seconds - stop program "{{ monit_home }}/stop_pcap_topology.sh" with timeout {{ topology_stop_timeout }} seconds - if status != 0 then restart - group pcap - group parsers - group metron - -check program yaf-parser with path "{{ monit_home }}/status_yaf_topology.sh" - start program "{{ monit_home }}/start_yaf_topology.sh" with timeout {{ topology_start_timeout }} seconds - stop program "{{ monit_home }}/stop_yaf_topology.sh" with timeout {{ topology_stop_timeout }} seconds - if status != 0 then restart - group yaf - group parsers - group metron - -check program bro-parser with path "{{ monit_home }}/status_bro_topology.sh" - start program "{{ monit_home }}/start_bro_topology.sh" with timeout {{ topology_start_timeout }} seconds - stop program "{{ monit_home }}/stop_bro_topology.sh" with timeout {{ topology_stop_timeout }} seconds - if status != 0 then restart - group bro - group parsers - group metron - -check program snort-parser with path "{{ monit_home }}/status_snort_topology.sh" - start program "{{ monit_home }}/start_snort_topology.sh" with timeout {{ topology_start_timeout }} seconds - stop program "{{ monit_home }}/stop_snort_topology.sh" with timeout {{ topology_stop_timeout }} seconds - if status != 0 then restart - group snort - group parsers - group metron diff --git a/metron-deployment/roles/monit/templates/scripts/start_enrichment_topology.sh b/metron-deployment/roles/monit/templates/scripts/start_enrichment_topology.sh deleted file mode 100644 index e170460e83..0000000000 --- a/metron-deployment/roles/monit/templates/scripts/start_enrichment_topology.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# start the enrichment topology -# -export METRON_VERSION={{ metron_version }} -export METRON_HOME={{ metron_directory }} -{{ item }} diff --git a/metron-deployment/roles/monit/templates/scripts/start_topology.sh b/metron-deployment/roles/monit/templates/scripts/start_topology.sh deleted file mode 100644 index 5e5286d375..0000000000 --- a/metron-deployment/roles/monit/templates/scripts/start_topology.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# start a storm topology -# -export METRON_VERSION={{ metron_version }} -export METRON_HOME={{ metron_directory }} -$METRON_HOME/bin/start_parser_topology.sh -k {{ kafka_broker_url }} -z {{ zookeeper_url }} -s {{ item }} diff --git a/metron-deployment/roles/monit/templates/scripts/status_topology.sh b/metron-deployment/roles/monit/templates/scripts/status_topology.sh deleted file mode 100644 index 67e9373b85..0000000000 --- a/metron-deployment/roles/monit/templates/scripts/status_topology.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# is a given storm topology running? -# - -TOPOLOGY={{ item }} -STATUS=`storm list | grep $TOPOLOGY | awk '{print $2}'` - -if [ "ACTIVE" = "$STATUS" ]; then - echo "Running: $TOPOLOGY" - exit 0 -else - echo "Stopped: $TOPOLOGY" - exit 1 -fi diff --git a/metron-deployment/roles/monit/templates/scripts/stop_topology.sh b/metron-deployment/roles/monit/templates/scripts/stop_topology.sh deleted file mode 100644 index d86462147c..0000000000 --- a/metron-deployment/roles/monit/templates/scripts/stop_topology.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# stop a storm topology -# -storm kill {{ item }} diff --git a/metron-deployment/roles/solr/defaults/main.yml b/metron-deployment/roles/solr/defaults/main.yml deleted file mode 100644 index b40d534b3b..0000000000 --- a/metron-deployment/roles/solr/defaults/main.yml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -rhel_hdp_utils_install_url: http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6/hdp-util.repo -solr_install_path: /opt/lucidworks-hdpsearch/solr -solr_user: solr -solr_collection_name: Metron -solr_config_dir: "{{ solr_install_path }}/server/solr/configsets/basic_configs/conf" -solr_bin_dir: "/opt/lucidworks-hdpsearch/solr/bin" -solr_config_name: "metron_conf" -solr_number_shards: "{{ groups['search'] | length }}" -solr_replication_factor: 1 -solr_autoSoftCommit_maxTime: 60 -solr_cmd: "{{ solr_bin_dir}}/solr create_collection -c {{ solr_collection_name }} -d {{ solr_config_dir }} -n {{ solr_config_name }} -shards {{ solr_number_shards }} -replicationFactor {{ solr_replication_factor }}" -hdp_utils_repo_path: /etc/yum.repos.d/HDP-UTILS.repo \ No newline at end of file diff --git a/metron-deployment/roles/solr/files/schema.xml b/metron-deployment/roles/solr/files/schema.xml deleted file mode 100644 index 43452a22bc..0000000000 --- a/metron-deployment/roles/solr/files/schema.xml +++ /dev/null @@ -1,191 +0,0 @@ - - - - - - - - ; - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/metron-deployment/roles/solr/meta/main.yml b/metron-deployment/roles/solr/meta/main.yml deleted file mode 100644 index 454dd3714c..0000000000 --- a/metron-deployment/roles/solr/meta/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -dependencies: - - ambari_gather_facts - - java_jdk - diff --git a/metron-deployment/roles/solr/tasks/main.yml b/metron-deployment/roles/solr/tasks/main.yml deleted file mode 100644 index cfbb6b5d77..0000000000 --- a/metron-deployment/roles/solr/tasks/main.yml +++ /dev/null @@ -1,74 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Check for Metron jar path - stat: path={{ hdp_utils_repo_path }} - register: hdp_utils - - -- name: Install HDP-UTILs Repo - get_url: - url: "{{ rhel_hdp_utils_install_url }}" - dest: /etc/yum.repos.d/HDP-UTILS.repo - when: hdp_utils.stat.exists == False - -- name: Install HDP-UTIL gpg key - rpm_key: - state: present - key: http://pgp.mit.edu/pks/lookup?op=get&search=0xB9733A7A07513CAD - when: hdp_utils.stat.exists == False - -- name: Install Solr - yum: - name: lucidworks-hdpsearch - state: present - -- name: Create solr.xml from template - template: - src: solr.xml - dest: "{{ solr_install_path }}/server/solr" - mode: 0644 - owner: "{{ solr_user }}" - group: "{{ solr_user }}" - -- name: Copy solrschema.xml to {{ inventory_hostname }} - copy: - src: schema.xml - dest: "{{ solr_config_dir }}" - mode: 0644 - owner: "{{ solr_user }}" - group: "{{ solr_user }}" - -- name: Create solrconfig.xml from template - template: - src: solrconfig.xml - dest: "{{ solr_config_dir }}" - mode: 0644 - owner: "{{ solr_user }}" - group: "{{ solr_user }}" - -- name: Start Solr - service: - name: solr - state: restarted - enabled: yes - -- name: Create Collection {{ solr_collection_name }} with {{ solr_number_shards }} shard(s) and replication factor {{ solr_replication_factor }} - shell: "{{ solr_cmd }}" - ignore_errors: yes - register: result - failed_when: result.rc == 1 and result.stderr.find("already exists!") == -1 diff --git a/metron-deployment/roles/solr/templates/solr.xml b/metron-deployment/roles/solr/templates/solr.xml deleted file mode 100644 index 407df13278..0000000000 --- a/metron-deployment/roles/solr/templates/solr.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - ${host:} - ${jetty.port:8983} - ${hostContext:solr} - - ${genericCoreNodeNames:true} - - {{ zookeeper_url }} - ${zkClientTimeout:30000} - ${distribUpdateSoTimeout:600000} - ${distribUpdateConnTimeout:60000} - - - - - ${socketTimeout:600000} - ${connTimeout:60000} - - - diff --git a/metron-deployment/roles/solr/templates/solrconfig.xml b/metron-deployment/roles/solr/templates/solrconfig.xml deleted file mode 100644 index b00af0f8c2..0000000000 --- a/metron-deployment/roles/solr/templates/solrconfig.xml +++ /dev/null @@ -1,583 +0,0 @@ - - - - - - - - - 5.2.1 - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - true - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - ${solr.autoSoftCommit.maxTime:{{ solr_autoSoftCommit_maxTime }}} - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - true - - - 20 - - - 200 - - - false - - - 2 - - - - - - - - - - - - - - - - - - - - explicit - 10 - - - - - - - - explicit - json - true - text - - - - - - - {!xport} - xsort - false - - - - query - - - - - - - text - - - - - - - - - - - - - - explicit - true - - - - - - - - - - - - - - true - false - - - terms - - - - - - *:* - - - diff --git a/metron-deployment/vagrant/full-dev-platform/Vagrantfile b/metron-deployment/vagrant/full-dev-platform/Vagrantfile index c73a70623a..7dbee12bc3 100644 --- a/metron-deployment/vagrant/full-dev-platform/Vagrantfile +++ b/metron-deployment/vagrant/full-dev-platform/Vagrantfile @@ -17,7 +17,7 @@ require 'getoptlong' ansibleTags='' -ansibleSkipTags='solr,sensors' +ansibleSkipTags='sensors' begin opts = GetoptLong.new( From 8b3d092e795b26bb0e82dd22bbf5f93651ddf7f5 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 21 Feb 2017 18:22:31 -0500 Subject: [PATCH 02/17] Working end to end. Auto-rpm deploy next. --- .../addon-services/METRON/CURRENT/role_command_order.json | 2 +- metron-deployment/playbooks/metron_install.yml | 7 ------- metron-deployment/roles/ambari_master/tasks/main.yml | 6 +++--- metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml | 3 +++ 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json index e08f401c86..7194845605 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json @@ -4,7 +4,7 @@ "general_deps" : { "_comment" : "dependencies for all cases", "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"], - "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"], + "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_ENRICHMENT_MASTER-START"], "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"], "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"], diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml index 6fc95e3a49..1a577b8ee1 100644 --- a/metron-deployment/playbooks/metron_install.yml +++ b/metron-deployment/playbooks/metron_install.yml @@ -65,13 +65,6 @@ tags: - sensor-stubs -- hosts: metron - become: true - roles: - - role: metron-rpms - tags: - - copy-rpms - # # monitor and start metron services with monit # diff --git a/metron-deployment/roles/ambari_master/tasks/main.yml b/metron-deployment/roles/ambari_master/tasks/main.yml index 935c48d90d..55c1d11739 100644 --- a/metron-deployment/roles/ambari_master/tasks/main.yml +++ b/metron-deployment/roles/ambari_master/tasks/main.yml @@ -44,13 +44,13 @@ - name: Copy MPack to Ambari Host copy: - src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-1.0.0.0-SNAPSHOT.tar.gz" + src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-0.3.1.0.tar.gz" dest: /tmp - name: Install MPack on Ambari Host - shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-1.0.0.0-SNAPSHOT.tar.gz + shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-0.3.1.0.tar.gz args: - creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-1.0.0.0/addon-services + creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-0.3.1.0/addon-services - name: start ambari server service: diff --git a/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml b/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml index d8e887dd13..ad734985e2 100644 --- a/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml +++ b/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml @@ -39,3 +39,6 @@ - "redef Kafka::topic_name = \"{{ bro_topic }}\";" - "redef Kafka::tag_json = T;" - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ kafka_broker_url }}\");" + +- name: Deploy bro configuration changes + shell: "{{ bro_home }}/bin/broctl deploy" \ No newline at end of file From 5865b404dd9bd10d18427ce94053d3183abbe89c Mon Sep 17 00:00:00 2001 From: David Lyle Date: Thu, 23 Feb 2017 17:10:10 -0500 Subject: [PATCH 03/17] Working on QD and AWS. --- metron-deployment/amazon-ec2/run.sh | 5 - .../inventory/full-dev-platform/hosts | 3 + .../package/scripts/dashboard/dashboard.p | 1686 ++++++++--------- .../package/scripts/enrichment_commands.py | 35 +- .../package/scripts/enrichment_master.py | 1 - .../package/scripts/params/params_linux.py | 4 +- .../package/scripts/params/status_params.py | 4 +- metron-deployment/playbooks/metron_build.yml | 21 + .../playbooks/metron_full_install.yml | 4 + .../roles/ambari_master/tasks/main.yml | 4 - .../roles/metron-builder/tasks/main.yml | 23 + 11 files changed, 928 insertions(+), 862 deletions(-) mode change 100755 => 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p create mode 100644 metron-deployment/playbooks/metron_build.yml create mode 100644 metron-deployment/roles/metron-builder/tasks/main.yml diff --git a/metron-deployment/amazon-ec2/run.sh b/metron-deployment/amazon-ec2/run.sh index cea45b38df..96c07af1d1 100755 --- a/metron-deployment/amazon-ec2/run.sh +++ b/metron-deployment/amazon-ec2/run.sh @@ -58,11 +58,6 @@ echo "=============================================================" >> $LOGFILE echo "Launching Metron[$ENV] @ $NOW"... >> $LOGFILE $DEPLOYDIR/../scripts/platform-info.sh >> $LOGFILE -# build metron -cd ../.. -mvn package -DskipTests -PHDP-2.5.0.0 -RC=$?; if [[ $RC != 0 ]]; then exit $RC; fi - # deploy metron cd $DEPLOYDIR export EC2_INI_PATH=conf/ec2.ini diff --git a/metron-deployment/inventory/full-dev-platform/hosts b/metron-deployment/inventory/full-dev-platform/hosts index 54a549b368..9bd9ea1e05 100644 --- a/metron-deployment/inventory/full-dev-platform/hosts +++ b/metron-deployment/inventory/full-dev-platform/hosts @@ -42,3 +42,6 @@ node1 [monit:children] sensors pcap_server + +[local] +127.0.0.1 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p old mode 100755 new mode 100644 index 8327eb8ad9..18d1315f42 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p @@ -9,21 +9,21 @@ Vvisualization p5 sV_id p6 -VWeb-Request-Type +VTop-Snort-Alerts-by-Source p7 sV_source p8 (dp9 VvisState p10 -V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}} p11 sVdescription p12 V sVtitle p13 -VWeb Request Type +VTop Snort Alerts by Source p14 sVuiStateJSON p15 @@ -32,509 +32,509 @@ p16 sVversion p17 I1 -sVsavedSearchId -p18 -Vweb-search -p19 -sVkibanaSavedObjectMeta -p20 -(dp21 -VsearchSourceJSON -p22 -V{"filter":[]} -p23 -sssV_index -p24 -V.kibana -p25 -sa(dp26 -V_score -p27 -F1 -sV_type -p28 -Vvisualization -p29 -sV_id -p30 -VTop-Snort-Alerts-by-Source -p31 -sV_source -p32 -(dp33 -VvisState -p34 -V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}} -p35 -sVdescription -p36 -V -sVtitle -p37 -VTop Snort Alerts by Source -p38 -sVuiStateJSON -p39 -V{} -p40 -sVversion -p41 -I1 sVkibanaSavedObjectMeta -p42 -(dp43 +p18 +(dp19 VsearchSourceJSON -p44 +p20 V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p45 +p21 sssV_index -p46 +p22 V.kibana -p47 -sa(dp48 +p23 +sa(dp24 V_score -p49 +p25 F1 sV_type -p50 +p26 Vvisualization -p51 +p27 sV_id -p52 +p28 VWelcome -p53 +p29 sV_source -p54 -(dp55 +p30 +(dp31 VvisState -p56 +p32 V{"title":"Welcome to Apache Metron","type":"markdown","params":{"markdown":"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\u005cn\u005cnApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\u005cn\u005cnThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron."},"aggs":[],"listeners":{}} -p57 +p33 sVdescription -p58 +p34 V sVtitle -p59 +p35 VWelcome to Apache Metron -p60 +p36 sVuiStateJSON -p61 +p37 V{} -p62 +p38 sVversion -p63 +p39 I1 sVkibanaSavedObjectMeta -p64 -(dp65 +p40 +(dp41 VsearchSourceJSON -p66 +p42 V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p67 +p43 sssV_index -p68 +p44 V.kibana -p69 -sa(dp70 +p45 +sa(dp46 V_score -p71 +p47 F1 sV_type -p72 +p48 Vsearch -p73 +p49 sV_id -p74 +p50 Vsnort-search -p75 +p51 sV_source -p76 -(dp77 +p52 +(dp53 Vsort -p78 -(lp79 +p54 +(lp55 Vtimestamp -p80 +p56 aVdesc -p81 +p57 asVhits -p82 +p58 I0 sVdescription -p83 +p59 V sVtitle -p84 +p60 VSnort Alerts -p85 +p61 sVversion -p86 +p62 I1 sVkibanaSavedObjectMeta -p87 -(dp88 +p63 +(dp64 VsearchSourceJSON -p89 +p65 V{"index":"snort*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p90 +p66 ssVcolumns -p91 -(lp92 +p67 +(lp68 Vmsg -p93 +p69 aVsig_id -p94 +p70 aVip_src_addr -p95 +p71 aVip_src_port -p96 +p72 aVip_dst_addr -p97 +p73 aVip_dst_port -p98 +p74 assV_index -p99 +p75 V.kibana -p100 -sa(dp101 +p76 +sa(dp77 V_score -p102 +p78 F1 sV_type -p103 +p79 Vsearch -p104 +p80 sV_id -p105 +p81 Vyaf-search -p106 +p82 sV_source -p107 -(dp108 +p83 +(dp84 Vsort -p109 -(lp110 +p85 +(lp86 Vtimestamp -p111 +p87 aVdesc -p112 +p88 asVhits -p113 +p89 I0 sVdescription -p114 +p90 V sVtitle -p115 +p91 VYAF -p116 +p92 sVversion -p117 +p93 I1 sVkibanaSavedObjectMeta -p118 -(dp119 +p94 +(dp95 VsearchSourceJSON -p120 +p96 V{"index":"yaf*","filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647},"query":{"query_string":{"query":"*","analyze_wildcard":true}}} -p121 +p97 ssVcolumns -p122 -(lp123 +p98 +(lp99 Vip_src_addr -p124 +p100 aVip_src_port -p125 +p101 aVip_dst_addr -p126 +p102 aVip_dst_port -p127 +p103 aVprotocol -p128 +p104 aVduration -p129 +p105 aVpkt -p130 +p106 assV_index -p131 +p107 V.kibana -p132 -sa(dp133 +p108 +sa(dp109 V_score -p134 +p110 F1 sV_type -p135 +p111 Vconfig -p136 +p112 sV_id -p137 +p113 V4.5.1 -p138 +p114 sV_source -p139 -(dp140 +p115 +(dp116 VbuildNum -p141 +p117 I9892 sVdefaultIndex -p142 -Vbro* -p143 -ssV_index -p144 -V.kibana -p145 -sa(dp146 -V_score -p147 -F1 -sV_type -p148 -Vindex-pattern -p149 -sV_id -p150 -Vbro* -p151 -sV_source -p152 -(dp153 -Vfields -p154 -V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p155 -sVtimeFieldName -p156 -Vtimestamp -p157 -sVtitle -p158 +p118 Vbro* -p159 +p119 ssV_index -p160 +p120 V.kibana -p161 -sa(dp162 +p121 +sa(dp122 V_score -p163 +p123 F1 sV_type -p164 +p124 Vvisualization -p165 +p125 sV_id -p166 -VFlow-Duration -p167 +p126 +VYAF-Flow(s) +p127 sV_source -p168 -(dp169 +p128 +(dp129 VvisState -p170 -V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}} -p171 +p130 +V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p131 sVdescription -p172 +p132 V sVtitle -p173 -VFlow Duration -p174 +p133 +VYAF Flows +p134 sVuiStateJSON -p175 -V{"vis":{"legendOpen":false}} -p176 +p135 +V{} +p136 sVversion -p177 +p137 I1 sVkibanaSavedObjectMeta -p178 -(dp179 +p138 +(dp139 VsearchSourceJSON -p180 +p140 V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p181 +p141 sssV_index -p182 +p142 V.kibana -p183 -sa(dp184 +p143 +sa(dp144 V_score -p185 +p145 F1 sV_type -p186 +p146 Vvisualization -p187 +p147 sV_id -p188 -VEvents -p189 +p148 +VTotal-Events +p149 sV_source -p190 -(dp191 +p150 +(dp151 VvisState -p192 -V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p193 +p152 +V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}} +p153 sVdescription -p194 +p154 V sVtitle -p195 -VEvents -p196 +p155 +VEvent Count +p156 sVuiStateJSON -p197 -V{"vis":{"legendOpen":false}} -p198 +p157 +V{} +p158 sVversion -p199 +p159 I1 sVkibanaSavedObjectMeta -p200 -(dp201 +p160 +(dp161 VsearchSourceJSON -p202 +p162 V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p203 +p163 sssV_index -p204 +p164 V.kibana -p205 -sa(dp206 +p165 +sa(dp166 V_score -p207 +p167 F1 sV_type -p208 +p168 Vvisualization -p209 +p169 sV_id -p210 -VWeb-Request-Header -p211 +p170 +VUnique-Location(s) +p171 sV_source -p212 -(dp213 +p172 +(dp173 VvisState -p214 -V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}} -p215 +p174 +V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}} +p175 sVdescription -p216 +p176 V sVtitle -p217 -VWeb Request Header -p218 +p177 +VGeo-IP Locations +p178 sVuiStateJSON -p219 +p179 V{} -p220 +p180 sVversion -p221 +p181 I1 sVkibanaSavedObjectMeta -p222 -(dp223 +p182 +(dp183 VsearchSourceJSON -p224 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p225 +p184 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p185 sssV_index -p226 +p186 V.kibana -p227 -sa(dp228 +p187 +sa(dp188 V_score -p229 +p189 F1 sV_type -p230 +p190 Vvisualization -p231 +p191 sV_id -p232 -VTop-Alerts-By-Host -p233 +p192 +VSnort-Alert-Types +p193 sV_source -p234 -(dp235 +p194 +(dp195 VvisState -p236 -V{"title":"New Visualization","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}} -p237 +p196 +V{"title":"Snort Alert Types","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"sig_id","customLabel":"Alert Type(s)"}}],"listeners":{}} +p197 sVdescription -p238 +p198 V sVtitle -p239 -VTop Alerts By Host -p240 +p199 +VSnort Alert Types +p200 sVuiStateJSON -p241 +p201 V{} -p242 +p202 sVversion -p243 +p203 +I1 +sVkibanaSavedObjectMeta +p204 +(dp205 +VsearchSourceJSON +p206 +V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p207 +sssV_index +p208 +V.kibana +p209 +sa(dp210 +V_score +p211 +F1 +sV_type +p212 +Vindex-pattern +p213 +sV_id +p214 +Vsnort* +p215 +sV_source +p216 +(dp217 +Vfields +p218 +V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +p219 +sVtimeFieldName +p220 +Vtimestamp +p221 +sVtitle +p222 +Vsnort* +p223 +ssV_index +p224 +V.kibana +p225 +sa(dp226 +V_score +p227 +F1 +sV_type +p228 +Vvisualization +p229 +sV_id +p230 +VDNS-Request(s) +p231 +sV_source +p232 +(dp233 +VvisState +p234 +V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p235 +sVdescription +p236 +V +sVtitle +p237 +VDNS Requests +p238 +sVuiStateJSON +p239 +V{} +p240 +sVversion +p241 I1 sVsavedSearchId -p244 -Vsnort-search -p245 +p242 +Vdns-search +p243 sVkibanaSavedObjectMeta -p246 -(dp247 +p244 +(dp245 VsearchSourceJSON -p248 +p246 V{"filter":[]} -p249 +p247 sssV_index -p250 +p248 V.kibana -p251 -sa(dp252 +p249 +sa(dp250 V_score -p253 +p251 F1 sV_type -p254 +p252 Vvisualization -p255 +p253 sV_id -p256 -VYAF-Flow(s) -p257 +p254 +VHTTP(S)-Requests +p255 sV_source -p258 -(dp259 +p256 +(dp257 VvisState -p260 -V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p261 +p258 +V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p259 sVdescription -p262 +p260 V sVtitle +p261 +VWeb Requests +p262 +sVuiStateJSON p263 -VYAF Flows +V{} p264 -sVuiStateJSON +sVversion p265 -V{} +I1 +sVsavedSearchId p266 -sVversion +Vweb-search p267 -I1 sVkibanaSavedObjectMeta p268 (dp269 VsearchSourceJSON p270 -V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +V{"filter":[]} p271 sssV_index p272 @@ -546,905 +546,905 @@ p275 F1 sV_type p276 -Vvisualization +Vsearch p277 sV_id p278 -VTop-DNS-Query +Vdns-search p279 sV_source p280 (dp281 -VvisState +Vsort p282 -V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} -p283 -sVdescription +(lp283 +Vtimestamp p284 -V -sVtitle +aVdesc p285 -VTop DNS Query +asVhits p286 -sVuiStateJSON +I0 +sVdescription p287 -V{} +V +sVtitle p288 -sVversion +VDNS Requests p289 +sVversion +p290 I1 sVkibanaSavedObjectMeta -p290 -(dp291 +p291 +(dp292 VsearchSourceJSON -p292 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p293 -sssV_index +V{"index":"bro*","query":{"query_string":{"query":"protocol: dns","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} p294 -V.kibana +ssVcolumns p295 -sa(dp296 -V_score +(lp296 +Vquery p297 -F1 -sV_type +aVqtype_name p298 -Vvisualization +aVanswers p299 -sV_id +aVip_src_addr p300 -VTotal-Events +aVip_dst_addr p301 -sV_source +assV_index p302 -(dp303 -VvisState -p304 -V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}} +V.kibana +p303 +sa(dp304 +V_score p305 -sVdescription +F1 +sV_type p306 -V -sVtitle +Vvisualization p307 -VEvent Count +sV_id p308 -sVuiStateJSON +VUnusual-Referrers p309 -V{} +sV_source p310 -sVversion -p311 -I1 -sVkibanaSavedObjectMeta +(dp311 +VvisState p312 -(dp313 -VsearchSourceJSON +V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}} +p313 +sVdescription p314 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +V +sVtitle p315 -sssV_index +VUnusual Referrers p316 -V.kibana +sVuiStateJSON p317 -sa(dp318 -V_score +V{} +p318 +sVversion p319 -F1 -sV_type +I1 +sVsavedSearchId p320 -Vvisualization +Vweb-search p321 -sV_id +sVkibanaSavedObjectMeta p322 -VEvent-Types -p323 -sV_source +(dp323 +VsearchSourceJSON p324 -(dp325 -VvisState +V{"filter":[]} +p325 +sssV_index p326 -V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} +V.kibana p327 -sVdescription -p328 -V -sVtitle +sa(dp328 +V_score p329 -VEvent Sources +F1 +sV_type p330 -sVuiStateJSON +Vvisualization p331 -V{} +sV_id p332 -sVversion +VFrequent-DNS-Requests p333 -I1 -sVkibanaSavedObjectMeta +sV_source p334 (dp335 -VsearchSourceJSON +VvisState p336 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1","customLabel":"DNS Query"}}],"listeners":{}} p337 -sssV_index +sVdescription p338 -V.kibana +V +sVtitle p339 -sa(dp340 -V_score +VFrequent DNS Requests +p340 +sVuiStateJSON p341 -F1 -sV_type +V{} p342 -Vvisualization +sVversion p343 -sV_id +I1 +sVkibanaSavedObjectMeta p344 -VUnique-Location(s) -p345 -sV_source +(dp345 +VsearchSourceJSON p346 -(dp347 -VvisState +V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p347 +sssV_index p348 -V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}} +V.kibana p349 -sVdescription -p350 -V -sVtitle +sa(dp350 +V_score p351 -VGeo-IP Locations +F1 +sV_type p352 -sVuiStateJSON +Vvisualization p353 -V{} +sV_id p354 -sVversion +VCountry p355 -I1 -sVkibanaSavedObjectMeta +sV_source p356 (dp357 -VsearchSourceJSON +VvisState p358 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +V{"title":"By Country","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"enrichments:geo:ip_src_addr:country","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} p359 -sssV_index +sVdescription p360 -V.kibana +V +sVtitle p361 -sa(dp362 -V_score +VBy Country +p362 +sVuiStateJSON p363 -F1 -sV_type +V{} p364 -Vvisualization +sVversion p365 -sV_id +I1 +sVkibanaSavedObjectMeta p366 -VSnort-Header -p367 -sV_source +(dp367 +VsearchSourceJSON p368 -(dp369 -VvisState +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p369 +sssV_index p370 -V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}} +V.kibana p371 -sVdescription -p372 -V -sVtitle +sa(dp372 +V_score p373 -VSnort +F1 +sV_type p374 -sVuiStateJSON +Vvisualization p375 -V{} +sV_id p376 -sVversion +VTop-Destinations p377 -I1 -sVkibanaSavedObjectMeta +sV_source p378 (dp379 -VsearchSourceJSON +VvisState p380 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +V{"title":"Top Destinations","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Destination IP"}}],"listeners":{}} p381 -sssV_index +sVdescription p382 -V.kibana +V +sVtitle p383 -sa(dp384 -V_score +VTop Destinations +p384 +sVuiStateJSON p385 -F1 -sV_type +V{} p386 -Vdashboard +sVversion p387 -sV_id +I1 +sVkibanaSavedObjectMeta p388 -VMetron-Dashboard -p389 -sV_source +(dp389 +VsearchSourceJSON p390 -(dp391 -Vhits +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p391 +sssV_index p392 -I0 -sVtimeRestore +V.kibana p393 -I00 -sVdescription -p394 -V -sVtitle +sa(dp394 +V_score p395 -VMetron Dashboard +F1 +sV_type p396 -sVuiStateJSON +Vvisualization p397 -V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}} +sV_id p398 -sVpanelsJSON +VYAF-Flows-Header p399 -V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","ip_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}] +sV_source p400 -sVoptionsJSON -p401 -V{"darkTheme":false} +(dp401 +VvisState p402 -sVversion +V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}} p403 -I1 -sVkibanaSavedObjectMeta +sVdescription p404 -(dp405 -VsearchSourceJSON +V +sVtitle +p405 +VYAF p406 -V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} +sVuiStateJSON p407 -sssV_index +V{} p408 -V.kibana +sVversion p409 -sa(dp410 -V_score -p411 -F1 -sV_type +I1 +sVkibanaSavedObjectMeta +p410 +(dp411 +VsearchSourceJSON p412 -Vvisualization +V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} p413 -sV_id +sssV_index p414 -VSnort-Alert-Types +V.kibana p415 -sV_source -p416 -(dp417 -VvisState +sa(dp416 +V_score +p417 +F1 +sV_type p418 -V{"title":"Snort Alert Types","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"sig_id","customLabel":"Alert Type(s)"}}],"listeners":{}} +Vvisualization p419 -sVdescription +sV_id p420 -V -sVtitle +VWeb-Request-Type p421 -VSnort Alert Types +sV_source p422 -sVuiStateJSON -p423 -V{} +(dp423 +VvisState p424 -sVversion +V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} p425 -I1 -sVkibanaSavedObjectMeta +sVdescription p426 -(dp427 -VsearchSourceJSON +V +sVtitle +p427 +VWeb Request Type p428 -V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +sVuiStateJSON p429 -sssV_index +V{} p430 -V.kibana +sVversion p431 -sa(dp432 -V_score +I1 +sVsavedSearchId +p432 +Vweb-search p433 -F1 -sV_type +sVkibanaSavedObjectMeta p434 -Vvisualization -p435 -sV_id +(dp435 +VsearchSourceJSON p436 -VFrequent-DNS-Queries +V{"filter":[]} p437 -sV_source +sssV_index p438 -(dp439 -VvisState -p440 -V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +V.kibana +p439 +sa(dp440 +V_score p441 -sVdescription +F1 +sV_type p442 -V -sVtitle +Vindex-pattern p443 -VFrequent DNS Requests +sV_id p444 -sVuiStateJSON +Vbro* p445 -V{} +sV_source p446 -sVversion -p447 -I1 -sVkibanaSavedObjectMeta +(dp447 +Vfields p448 -(dp449 -VsearchSourceJSON +V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +p449 +sVtimeFieldName p450 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +Vtimestamp p451 -sssV_index +sVtitle p452 -V.kibana +Vbro* p453 -sa(dp454 -V_score +ssV_index +p454 +V.kibana p455 +sa(dp456 +V_score +p457 F1 sV_type -p456 -Vvisualization -p457 -sV_id p458 -VLocation-Header +Vvisualization p459 -sV_source +sV_id p460 -(dp461 -VvisState +VFlow-Duration +p461 +sV_source p462 -V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}} -p463 -sVdescription +(dp463 +VvisState p464 -V -sVtitle +V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}} p465 -VEnrichment +sVdescription p466 -sVuiStateJSON +V +sVtitle p467 -V{} +VFlow Duration p468 -sVversion +sVuiStateJSON p469 +V{"vis":{"legendOpen":false}} +p470 +sVversion +p471 I1 sVkibanaSavedObjectMeta -p470 -(dp471 -VsearchSourceJSON p472 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p473 -sssV_index +(dp473 +VsearchSourceJSON p474 -V.kibana +V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p475 -sa(dp476 -V_score +sssV_index +p476 +V.kibana p477 +sa(dp478 +V_score +p479 F1 sV_type -p478 -Vsearch -p479 -sV_id p480 -Vweb-search +Vvisualization p481 -sV_source +sV_id p482 -(dp483 -Vsort +VEvents +p483 +sV_source p484 -(lp485 -Vtimestamp +(dp485 +VvisState p486 -aVdesc +V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} p487 -asVhits -p488 -I0 sVdescription -p489 +p488 V sVtitle +p489 +VEvents p490 -VWeb Requests +sVuiStateJSON p491 -sVversion +V{"vis":{"legendOpen":false}} p492 +sVversion +p493 I1 sVkibanaSavedObjectMeta -p493 -(dp494 +p494 +(dp495 VsearchSourceJSON -p495 -V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} p496 -ssVcolumns +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p497 -(lp498 -Vmethod +sssV_index +p498 +V.kibana p499 -aVhost -p500 -aVuri +sa(dp500 +V_score p501 -aVreferrer +F1 +sV_type p502 -aVip_src_addr +Vvisualization p503 -aVip_dst_addr +sV_id p504 -assV_index +VWeb-Request-Header p505 -V.kibana +sV_source p506 -sa(dp507 -V_score +(dp507 +VvisState p508 -F1 -sV_type +V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}} p509 -Vindex-pattern +sVdescription p510 -sV_id +V +sVtitle p511 -Vsnort* +VWeb Request Header p512 -sV_source +sVuiStateJSON p513 -(dp514 -Vfields +V{} +p514 +sVversion p515 -V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +I1 +sVkibanaSavedObjectMeta p516 -sVtimeFieldName -p517 -Vtimestamp +(dp517 +VsearchSourceJSON p518 -sVtitle +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p519 -Vsnort* +sssV_index p520 -ssV_index -p521 V.kibana -p522 -sa(dp523 +p521 +sa(dp522 V_score -p524 +p523 F1 sV_type +p524 +Vvisualization p525 -Vindex-pattern -p526 sV_id +p526 +VTop-Alerts-By-Host p527 -Vyaf* -p528 sV_source -p529 -(dp530 -Vfields +p528 +(dp529 +VvisState +p530 +V{"title":"New Visualization","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}} p531 -V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +sVdescription p532 -sVtimeFieldName +V +sVtitle p533 -Vtimestamp +VTop Alerts By Host p534 -sVtitle +sVuiStateJSON p535 -Vyaf* +V{} p536 -ssV_index +sVversion p537 -V.kibana +I1 +sVsavedSearchId p538 -sa(dp539 -V_score +Vsnort-search +p539 +sVkibanaSavedObjectMeta p540 -F1 -sV_type -p541 -Vvisualization +(dp541 +VsearchSourceJSON p542 -sV_id +V{"filter":[]} p543 -VDNS-Request(s) +sssV_index p544 -sV_source +V.kibana p545 -(dp546 -VvisState +sa(dp546 +V_score p547 -V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +F1 +sV_type p548 -sVdescription +Vvisualization p549 -V -sVtitle +sV_id p550 -VDNS Requests +VTop-DNS-Query p551 -sVuiStateJSON +sV_source p552 -V{} -p553 -sVversion +(dp553 +VvisState p554 -I1 -sVsavedSearchId +V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} p555 -Vdns-search +sVdescription p556 -sVkibanaSavedObjectMeta +V +sVtitle p557 -(dp558 -VsearchSourceJSON +VTop DNS Query +p558 +sVuiStateJSON p559 -V{"filter":[]} +V{} p560 -sssV_index +sVversion p561 -V.kibana +I1 +sVkibanaSavedObjectMeta p562 -sa(dp563 -V_score +(dp563 +VsearchSourceJSON p564 -F1 -sV_type +V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p565 -Vvisualization +sssV_index p566 -sV_id +V.kibana p567 -VHTTP(S)-Requests -p568 -sV_source +sa(dp568 +V_score p569 -(dp570 -VvisState +F1 +sV_type +p570 +Vvisualization p571 -V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +sV_id p572 -sVdescription +VEvent-Types p573 -V -sVtitle +sV_source p574 -VWeb Requests -p575 -sVuiStateJSON +(dp575 +VvisState p576 -V{} +V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} p577 -sVversion +sVdescription p578 -I1 -sVsavedSearchId +V +sVtitle p579 -Vweb-search +VEvent Sources p580 -sVkibanaSavedObjectMeta +sVuiStateJSON p581 -(dp582 -VsearchSourceJSON +V{} +p582 +sVversion p583 -V{"filter":[]} +I1 +sVkibanaSavedObjectMeta p584 +(dp585 +VsearchSourceJSON +p586 +V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p587 sssV_index -p585 +p588 V.kibana -p586 -sa(dp587 +p589 +sa(dp590 V_score -p588 +p591 F1 sV_type -p589 -Vsearch -p590 -sV_id -p591 -Vdns-search p592 -sV_source +Vvisualization p593 -(dp594 -Vsort +sV_id +p594 +VSnort-Header p595 -(lp596 -Vtimestamp -p597 -aVdesc +sV_source +p596 +(dp597 +VvisState p598 -asVhits +V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}} p599 -I0 sVdescription p600 V sVtitle p601 -VDNS Requests +VSnort p602 -sVversion +sVuiStateJSON p603 +V{} +p604 +sVversion +p605 I1 sVkibanaSavedObjectMeta -p604 -(dp605 -VsearchSourceJSON p606 -V{"index":"bro*","query":{"query_string":{"query":"protocol: dns","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p607 -ssVcolumns +(dp607 +VsearchSourceJSON p608 -(lp609 -Vquery +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p609 +sssV_index p610 -aVqtype_name +V.kibana p611 -aVanswers -p612 -aVip_src_addr +sa(dp612 +V_score p613 -aVip_dst_addr +F1 +sV_type p614 -assV_index +Vdashboard p615 -V.kibana +sV_id p616 -sa(dp617 -V_score +VMetron-Dashboard +p617 +sV_source p618 -F1 -sV_type -p619 -Vvisualization +(dp619 +Vhits p620 -sV_id +I0 +sVtimeRestore p621 -VFlow-Locations +I00 +sVdescription p622 -sV_source +V +sVtitle p623 -(dp624 -VvisState +VMetron Dashboard +p624 +sVuiStateJSON p625 -V{"title":"New Visualization","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":false,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}} +V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}} p626 -sVdescription +sVpanelsJSON p627 -V -sVtitle +V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","ip_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}] p628 -VFlow Locations +sVoptionsJSON p629 -sVuiStateJSON +V{"darkTheme":false} p630 -V{} -p631 sVversion -p632 +p631 I1 sVkibanaSavedObjectMeta -p633 -(dp634 +p632 +(dp633 VsearchSourceJSON +p634 +V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} p635 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p636 sssV_index -p637 +p636 V.kibana -p638 -sa(dp639 +p637 +sa(dp638 V_score -p640 +p639 F1 sV_type -p641 +p640 Vvisualization -p642 +p641 sV_id +p642 +VFrequent-DNS-Queries p643 -VUnusual-Referrers -p644 sV_source -p645 -(dp646 +p644 +(dp645 VvisState +p646 +V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} p647 -V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}} -p648 sVdescription -p649 +p648 V sVtitle +p649 +VFrequent DNS Requests p650 -VUnusual Referrers -p651 sVuiStateJSON -p652 +p651 V{} -p653 +p652 sVversion -p654 +p653 I1 -sVsavedSearchId -p655 -Vweb-search -p656 sVkibanaSavedObjectMeta -p657 -(dp658 +p654 +(dp655 VsearchSourceJSON -p659 -V{"filter":[]} -p660 +p656 +V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p657 sssV_index -p661 +p658 V.kibana -p662 -sa(dp663 +p659 +sa(dp660 V_score -p664 +p661 F1 sV_type -p665 +p662 Vvisualization -p666 +p663 sV_id -p667 -VFrequent-DNS-Requests -p668 +p664 +VLocation-Header +p665 sV_source -p669 -(dp670 +p666 +(dp667 VvisState -p671 -V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1","customLabel":"DNS Query"}}],"listeners":{}} -p672 +p668 +V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}} +p669 sVdescription -p673 +p670 V sVtitle -p674 -VFrequent DNS Requests -p675 +p671 +VEnrichment +p672 sVuiStateJSON -p676 +p673 V{} -p677 +p674 sVversion -p678 +p675 I1 sVkibanaSavedObjectMeta -p679 -(dp680 +p676 +(dp677 VsearchSourceJSON -p681 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p682 +p678 +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p679 sssV_index -p683 +p680 V.kibana -p684 -sa(dp685 +p681 +sa(dp682 V_score -p686 +p683 F1 sV_type +p684 +Vsearch +p685 +sV_id +p686 +Vweb-search p687 -Vvisualization +sV_source p688 -sV_id -p689 -VCountry +(dp689 +Vsort p690 -sV_source -p691 -(dp692 -VvisState +(lp691 +Vtimestamp +p692 +aVdesc p693 -V{"title":"By Country","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"enrichments:geo:ip_src_addr:country","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +asVhits p694 +I0 sVdescription p695 V sVtitle p696 -VBy Country +VWeb Requests p697 -sVuiStateJSON -p698 -V{} -p699 sVversion -p700 +p698 I1 sVkibanaSavedObjectMeta -p701 -(dp702 +p699 +(dp700 VsearchSourceJSON +p701 +V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} +p702 +ssVcolumns p703 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p704 -sssV_index +(lp704 +Vmethod p705 -V.kibana +aVhost p706 -sa(dp707 -V_score +aVuri +p707 +aVreferrer p708 -F1 -sV_type +aVip_src_addr p709 -Vvisualization +aVip_dst_addr p710 -sV_id +assV_index p711 -VTop-Destinations +V.kibana p712 -sV_source -p713 -(dp714 -VvisState +sa(dp713 +V_score +p714 +F1 +sV_type p715 -V{"title":"Top Destinations","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Destination IP"}}],"listeners":{}} +Vindex-pattern p716 -sVdescription +sV_id p717 -V -sVtitle +Vyaf* p718 -VTop Destinations +sV_source p719 -sVuiStateJSON -p720 -V{} +(dp720 +Vfields p721 -sVversion +V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] p722 -I1 -sVkibanaSavedObjectMeta +sVtimeFieldName p723 -(dp724 -VsearchSourceJSON +Vtimestamp +p724 +sVtitle p725 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +Vyaf* p726 -sssV_index +ssV_index p727 V.kibana p728 @@ -1502,21 +1502,21 @@ Vvisualization p754 sV_id p755 -VYAF-Flows-Header +VFlow-Locations p756 sV_source p757 (dp758 VvisState p759 -V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}} +V{"title":"Flow Locations","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":true,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}} p760 sVdescription p761 V sVtitle p762 -VYAF +VFlow Locations p763 sVuiStateJSON p764 @@ -1530,7 +1530,7 @@ p767 (dp768 VsearchSourceJSON p769 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} +V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p770 sssV_index p771 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index 87c78e23f2..4523198949 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -30,7 +30,9 @@ class EnrichmentCommands: __enrichment_topic = None __enrichment_error_topic = None __threat_intel_error_topic = None - __configured = False + __kafka_configured = False + __hbase_configured = False + __geo_configured = False def __init__(self, params): if params is None: @@ -40,14 +42,30 @@ def __init__(self, params): self.__enrichment_topic = params.metron_enrichment_topic self.__enrichment_error_topic = params.metron_enrichment_error_topic self.__threat_intel_error_topic = params.metron_threat_intel_error_topic - self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file) + self.__kafka_configured = os.path.isfile(self.__params.enrichment_kafka_configured_flag_file) + self.__hbase_configured = os.path.isfile(self.__params.enrichment_hbase_configured_flag_file) + self.__geo_configured = os.path.isfile(self.__params.enrichment_geo_configured_flag_file) - def is_configured(self): - return self.__configured + def is_kafka_configured(self): + return self.__kafka_configured - def set_configured(self): - Logger.info("Setting Configured to True") - File(self.__params.enrichment_configured_flag_file, + def set_kafka_configured(self): + Logger.info("Setting Kafka Configured to True") + File(self.__params.enrichment_kafka_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + + def set_hbase_configured(self): + Logger.info("Setting HBase Configured to True") + File(self.__params.enrichment_hbase_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + + def set_geo_configured(self): + Logger.info("Setting GEO Configured to True") + File(self.__params.enrichment_geo_configured_flag_file, content="", owner=self.__params.metron_user, mode=0775) @@ -106,6 +124,7 @@ def init_geo(self): Logger.info("Executing command " + command) Execute(command, user=self.__params.metron_user, tries=1, logoutput=True) Logger.info("Done intializing GeoIP data") + self.set_geo_configured() def init_kafka_topics(self): Logger.info('Creating Kafka topics') @@ -133,6 +152,7 @@ def init_kafka_topics(self): retention_bytes)) Logger.info("Done creating Kafka topics") + self.set_kafka_configured() def start_enrichment_topology(self): Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology)) @@ -198,3 +218,4 @@ def create_hbase_tables(self): path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' ) Logger.info("Done creating HBase Tables") + self.set_hbase_configured() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index 71d80c88a7..cced92c709 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -54,7 +54,6 @@ def start(self, env, upgrade_type=None): commands.init_kafka_topics() commands.create_hbase_tables() commands.init_geo() - commands.set_configured() commands.start_enrichment_topology() diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index a0446e417b..a802e452ae 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -51,7 +51,9 @@ metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir metron_zookeeper_config_path = status_params.metron_zookeeper_config_path parsers_configured_flag_file = status_params.parsers_configured_flag_file -enrichment_configured_flag_file = status_params.enrichment_configured_flag_file +enrichment_kafka_configured_flag_file = status_params.enrichment_kafka_configured_flag_file +enrichment_hbase_configured_flag_file = status_params.enrichment_hbase_configured_flag_file +enrichment_geo_configured_flag_file = status_params.enrichment_geo_configured_flag_file indexing_configured_flag_file = status_params.indexing_configured_flag_file global_json_template = config['configurations']['metron-env']['global-json'] global_properties_template = config['configurations']['metron-env']['elasticsearch-properties'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index e8a8568e4c..67213f9f80 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -47,7 +47,9 @@ indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' # Enrichment -enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_configured' +enrichment_kafka_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_configured' +enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_configured' +enrichment_geo_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_geo_configured' # Storm storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] diff --git a/metron-deployment/playbooks/metron_build.yml b/metron-deployment/playbooks/metron_build.yml new file mode 100644 index 0000000000..70d2ae513e --- /dev/null +++ b/metron-deployment/playbooks/metron_build.yml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- hosts: all + become: false + roles: + - role: metron-builder diff --git a/metron-deployment/playbooks/metron_full_install.yml b/metron-deployment/playbooks/metron_full_install.yml index 97e5084dd6..3f1fa9ac29 100644 --- a/metron-deployment/playbooks/metron_full_install.yml +++ b/metron-deployment/playbooks/metron_full_install.yml @@ -15,6 +15,10 @@ # limitations under the License. # --- +- include: metron_build.yml + tags: + - build + - include: ambari_install.yml tags: - ambari diff --git a/metron-deployment/roles/ambari_master/tasks/main.yml b/metron-deployment/roles/ambari_master/tasks/main.yml index 55c1d11739..ee185c6094 100644 --- a/metron-deployment/roles/ambari_master/tasks/main.yml +++ b/metron-deployment/roles/ambari_master/tasks/main.yml @@ -38,10 +38,6 @@ register: ambari_server_setup failed_when: ambari_server_setup.stderr -- name: Build MPack - local_action: shell cd {{ playbook_dir }}/.. && mvn clean package - become: false - - name: Copy MPack to Ambari Host copy: src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-0.3.1.0.tar.gz" diff --git a/metron-deployment/roles/metron-builder/tasks/main.yml b/metron-deployment/roles/metron-builder/tasks/main.yml new file mode 100644 index 0000000000..b028524dca --- /dev/null +++ b/metron-deployment/roles/metron-builder/tasks/main.yml @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Build Deployment Artifacts + local_action: shell cd {{ playbook_dir }}/../.. && mvn clean package -DskipTests -P HDP-2.5.0.0,build-rpms + become: false + run_once: true + + From 71f4766a56e8a5d1737e6469445a856721b49fac Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 28 Feb 2017 15:18:52 -0500 Subject: [PATCH 04/17] Added extra module to reinstall Metron services in support of Quick-dev. --- .../extra_modules/ambari_service_state.py | 352 ++++++++++++++++++ .../playbooks/metron_install.yml | 5 + .../roles/quick_dev/tasks/main.yml | 82 ++++ metron-platform/metron-enrichment/pom.xml | 20 +- 4 files changed, 454 insertions(+), 5 deletions(-) create mode 100644 metron-deployment/extra_modules/ambari_service_state.py create mode 100644 metron-deployment/roles/quick_dev/tasks/main.yml diff --git a/metron-deployment/extra_modules/ambari_service_state.py b/metron-deployment/extra_modules/ambari_service_state.py new file mode 100644 index 0000000000..e67bcfe406 --- /dev/null +++ b/metron-deployment/extra_modules/ambari_service_state.py @@ -0,0 +1,352 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +DOCUMENTATION = ''' +--- +module: ambari_service_state +version_added: "2.1" +author: Apache Metron (Incubating : https://github.com/apache/incubator-metron ) +short_description: Start/Stop/Change Service or Component State +description: + - Start/Stop/Change Service or Component State +options: + host: + description: + The hostname for the ambari web server + port: + description: + The port for the ambari web server + username: + description: + The username for the ambari web server + password: + description: + The name of the cluster in web server + required: yes + cluster_name: + description: + The name of the cluster in ambari + required: yes + service_name: + description: + The name of the service to alter + required: no + component_name: + description: + The name of the component to alter + required: no + component_host: + description: + The host running the targeted component. Required when component_name is used. + required: no + state: + description: + The desired service/component state. + wait_for_complete: + description: + Whether to wait for the request to complete before returning. Default is False. + required: no + requirements: [ 'requests'] +''' + +EXAMPLES = ''' +# must use full relative path to any files in stored in roles/role_name/files/ +- name: Create a new ambari cluster + ambari_cluster_state: + host: localhost + port: 8080 + username: admin + password: admin + cluster_name: my_cluster + cluster_state: present + blueprint_var: roles/my_role/files/blueprint.yml + blueprint_name: hadoop + wait_for_complete: True +- name: Start the ambari cluster + ambari_cluster_state: + host: localhost + port: 8080 + username: admin + password: admin + cluster_name: my_cluster + cluster_state: started + wait_for_complete: True +- name: Stop the ambari cluster + ambari_cluster_state: + host: localhost + port: 8080 + username: admin + password: admin + cluster_name: my_cluster + cluster_state: stopped + wait_for_complete: True +- name: Delete the ambari cluster + ambari_cluster_state: + host: localhost + port: 8080 + username: admin + password: admin + cluster_name: my_cluster + cluster_state: absent +''' + +RETURN = ''' +results: + description: The content of the requests object returned from the RESTful call + returned: success + type: string +''' + +__author__ = 'apachemetron' + +import json + +try: + import requests +except ImportError: + REQUESTS_FOUND = False +else: + REQUESTS_FOUND = True + + +def main(): + + argument_spec = dict( + host=dict(type='str', default=None, required=True), + port=dict(type='int', default=None, required=True), + username=dict(type='str', default=None, required=True), + password=dict(type='str', default=None, required=True), + cluster_name=dict(type='str', default=None, required=True), + state=dict(type='str', default=None, required=True, + choices=['started', 'stopped', 'deleted']), + service_name=dict(type='str', required=False), + component_name=dict(type='str', default=None, required=False), + component_host=dict(type='str', default=None, required=False), + wait_for_complete=dict(default=False, required=False, choices=BOOLEANS), + ) + + required_together = ['component_name', 'component_host'] + + module = AnsibleModule( + argument_spec=argument_spec, + required_together=required_together + ) + + if not REQUESTS_FOUND: + module.fail_json( + msg='requests library is required for this module') + + p = module.params + + host = p.get('host') + port = p.get('port') + username = p.get('password') + password = p.get('password') + cluster_name = p.get('cluster_name') + state = p.get('state') + service_name = p.get('service_name') + component_name = p.get('component_name') + component_host = p.get('component_host') + wait_for_complete = p.get('wait_for_complete') + component_mode = False + ambari_url = 'http://{0}:{1}'.format(host, port) + + if component_name: + component_mode = True + + try: + if not cluster_exists(ambari_url, username, password, cluster_name): + module.fail_json(msg="Cluster name {0} does not exist".format(cluster_name)) + + if state in ['started', 'stopped', 'installed']: + desired_state = '' + + if state == 'started': + desired_state = 'STARTED' + elif state in ['stopped','installed']: + desired_state = 'INSTALLED' + + if component_mode: + if desired_state == 'INSTALLED': + if(can_add_component(ambari_url, username, password, cluster_name, component_name, component_host)): + add_component_to_host(ambari_url, username, password, cluster_name, component_name, component_host) + request = set_component_state(ambari_url, username, password, cluster_name, component_name, component_host, desired_state) + else: + request = set_service_state(ambari_url,username,password,cluster_name,service_name, desired_state) + if wait_for_complete: + try: + request_id = json.loads(request.content)['Requests']['id'] + except ValueError: + module.exit_json(changed=True, results=request.content) + status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2) + if status != 'COMPLETED': + module.fail_json(msg="Request failed with status {0}".format(status)) + module.exit_json(changed=True, results=request.content) + + elif state == 'deleted': + if component_mode: + request = delete_component(ambari_url, username, password, cluster_name, component_name, component_host) + else: + request = delete_service(ambari_url,username,password,cluster_name,service_name) + module.exit_json(changed=True, results=request.content) + + except requests.ConnectionError, e: + module.fail_json(msg="Could not connect to Ambari client: " + str(e.message)) + except Exception, e: + module.fail_json(msg="Ambari client exception occurred: " + str(e.message)) + + +def get_clusters(ambari_url, user, password): + r = get(ambari_url, user, password, '/api/v1/clusters') + if r.status_code != 200: + msg = 'Could not get cluster list: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + clusters = json.loads(r.content) + return clusters['items'] + + +def cluster_exists(ambari_url, user, password, cluster_name): + clusters = get_clusters(ambari_url, user, password) + return cluster_name in [item['Clusters']['cluster_name'] for item in clusters] + + +def get_request_status(ambari_url, user, password, cluster_name, request_id): + path = '/api/v1/clusters/{0}/requests/{1}'.format(cluster_name, request_id) + r = get(ambari_url, user, password, path) + if r.status_code != 200: + msg = 'Could not get cluster request status: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + service = json.loads(r.content) + return service['Requests']['request_status'] + + +def wait_for_request_complete(ambari_url, user, password, cluster_name, request_id, sleep_time): + while True: + status = get_request_status(ambari_url, user, password, cluster_name, request_id) + if status == 'COMPLETED': + return status + elif status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'SKIPPED_FAILED']: + return status + else: + time.sleep(sleep_time) + + +def set_service_state(ambari_url, user, password, cluster_name, service_name, desired_state): + path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name) + request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(service_name,desired_state)}, + "Body": {"ServiceInfo": {"state": "{0}".format(desired_state)}}} + payload = json.dumps(request) + r = put(ambari_url, user, password, path, payload) + if r.status_code not in [202, 200]: + msg = 'Could not set service state: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def set_component_state(ambari_url, user, password, cluster_name, component_name, component_host, desired_state): + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(component_name,desired_state)}, + "Body": {"HostRoles": {"state": "{0}".format(desired_state)}}} + payload = json.dumps(request) + r = put(ambari_url, user, password, path, payload) + if r.status_code not in [202, 200]: + msg = 'Could not set component state: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def delete_component(ambari_url, user, password, cluster_name, component_name, component_host): + enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host) + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + r = delete(ambari_url,user,password,path) + if r.status_code not in [202, 200]: + msg = 'Could not set service state: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host): + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + request = {"RequestInfo":{"context":"Turn On Maintenance Mode for {0}".format(component_name)}, + "Body":{"HostRoles":{"maintenance_state":"ON"}}} + payload = json.dumps(request) + r = put(ambari_url, user, password, path, payload) + if r.status_code not in [202, 200]: + msg = 'Could not set maintenance mode: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def delete_service(ambari_url, user, password, cluster_name, service_name): + path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name) + r = delete(ambari_url,user,password,path) + if r.status_code not in [202, 200]: + msg = 'Could not delete service: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def add_component_to_host(ambari_url, user, password, cluster_name, component_name, component_host): + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + r = post(ambari_url, user, password, path,'') + if r.status_code not in [202,201,200]: + msg = 'Could not add {0} to host {1}: request code {2}, \ + request message {3}'.format(component_name,component_host,r.status_code, r.content) + raise Exception(msg) + return r + + +def can_add_component(ambari_url, user, password, cluster_name, component_name, component_host): + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + r = get(ambari_url, user, password, path) + return r.status_code == 404 + + +def get(ambari_url, user, password, path): + r = requests.get(ambari_url + path, auth=(user, password)) + return r + + +def put(ambari_url, user, password, path, data): + headers = {'X-Requested-By': 'ambari'} + r = requests.put(ambari_url + path, data=data, auth=(user, password), headers=headers) + return r + + +def post(ambari_url, user, password, path, data): + headers = {'X-Requested-By': 'ambari'} + r = requests.post(ambari_url + path, data=data, auth=(user, password), headers=headers) + return r + + +def delete(ambari_url, user, password, path): + headers = {'X-Requested-By': 'ambari'} + r = requests.delete(ambari_url + path, auth=(user, password), headers=headers) + return r + + +from ansible.module_utils.basic import * +if __name__ == '__main__': + main() diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml index 1a577b8ee1..f8f9bf017a 100644 --- a/metron-deployment/playbooks/metron_install.yml +++ b/metron-deployment/playbooks/metron_install.yml @@ -82,3 +82,8 @@ become: false roles: - { role: deployment-report, tags: ['report'] } + +- hosts: metron + become: true + roles: + - { role: quick_dev, tags: ['quick_dev']} \ No newline at end of file diff --git a/metron-deployment/roles/quick_dev/tasks/main.yml b/metron-deployment/roles/quick_dev/tasks/main.yml new file mode 100644 index 0000000000..9825e71adc --- /dev/null +++ b/metron-deployment/roles/quick_dev/tasks/main.yml @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Shut Down Metron + ambari_service_state: + host: "{{ groups.ambari_master[0] }}" + port: "{{ ambari_port }}" + username: "{{ ambari_user }}" + password: "{{ ambari_password }}" + cluster_name: "{{ cluster_name }}" + state: stopped + service_name: METRON + wait_for_complete: True + +- name: Delete the Metron Components from Ambari + ambari_service_state: + host: "{{ groups.ambari_master[0] }}" + port: "{{ ambari_port }}" + username: "{{ ambari_user }}" + password: "{{ ambari_password }}" + cluster_name: "{{ cluster_name }}" + state: deleted + component_name: "{{ item }}" + component_host: "{{ inventory_hostname }}" + with_items: + - METRON_ENRICHMENT_MASTER + - METRON_INDEXING + - METRON_PARSERS + +- name: Remove the Metron packages + package: + name: "{{ item }}" + state: absent + with_items: + - metron-common + - metron-data-management + - metron-parsers + - metron-enrichment + - metron-indexing + - metron-elasticsearch + +- name: Re-install the Metron Packages via Ambari + ambari_service_state: + host: "{{ groups.ambari_master[0] }}" + port: "{{ ambari_port }}" + username: "{{ ambari_user }}" + password: "{{ ambari_password }}" + cluster_name: "{{ cluster_name }}" + state: stopped + component_name: "{{ item }}" + component_host: "{{ inventory_hostname }}" + wait_for_complete: True + with_items: + - METRON_ENRICHMENT_MASTER + - METRON_INDEXING + - METRON_PARSERS + +- name: Start Metron + ambari_service_state: + host: "{{ groups.ambari_master[0] }}" + port: "{{ ambari_port }}" + username: "{{ ambari_user }}" + password: "{{ ambari_password }}" + cluster_name: "{{ cluster_name }}" + state: started + service_name: METRON + wait_for_complete: True + diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index 4dca431a66..af09497025 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -73,11 +73,21 @@ provided - com.maxmind.geoip2 - geoip2 - ${geoip.version} - - + com.maxmind.geoip2 + geoip2 + ${geoip.version} + + + jackson-core + com.fasterxml.jackson.core + + + jackson-databind + com.fasterxml.jackson.core + + + + org.apache.hbase hbase-client ${global_hbase_version} From b3e0ba885c565b6c26bc8f258e1a33e7acffe0a4 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 28 Feb 2017 16:38:53 -0500 Subject: [PATCH 05/17] Changed the base image iso url and updated the skip-tags for full dev. --- metron-deployment/packer-build/hdp-centos-6.7.json | 2 +- metron-deployment/vagrant/full-dev-platform/Vagrantfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/packer-build/hdp-centos-6.7.json b/metron-deployment/packer-build/hdp-centos-6.7.json index 76efc8ea9a..ec28b1af2c 100644 --- a/metron-deployment/packer-build/hdp-centos-6.7.json +++ b/metron-deployment/packer-build/hdp-centos-6.7.json @@ -122,7 +122,7 @@ "ks_path": "centos-6.7/ks.cfg", "memory": "8192", "metadata": "floppy/dummy_metadata.json", - "mirror": "http://mirrors.kernel.org/centos", + "mirror": "http://archive.kernel.org/centos-vault", "mirror_directory": "6.7/isos/x86_64", "name": "hdp-centos-6.7", "no_proxy": "{{env `no_proxy`}}", diff --git a/metron-deployment/vagrant/full-dev-platform/Vagrantfile b/metron-deployment/vagrant/full-dev-platform/Vagrantfile index 7dbee12bc3..47165167ae 100644 --- a/metron-deployment/vagrant/full-dev-platform/Vagrantfile +++ b/metron-deployment/vagrant/full-dev-platform/Vagrantfile @@ -17,7 +17,7 @@ require 'getoptlong' ansibleTags='' -ansibleSkipTags='sensors' +ansibleSkipTags='sensors,quick_dev' begin opts = GetoptLong.new( From 3bf9577c84d9f4ec57aad60274e921a5e233bfd5 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Wed, 1 Mar 2017 19:29:18 -0500 Subject: [PATCH 06/17] Fixes to make the Packer quick dev image build. --- .../package/scripts/enrichment_commands.py | 6 ++++++ .../package/scripts/enrichment_master.py | 4 +++- .../packer-build/ansible/playbook.yml | 8 -------- .../packer-build/ansible/tasks/cleanup.yml | 7 ------- .../ansible/tasks/clear_topics.yml | 1 + .../ansible/tasks/remove_services.yml | 3 +++ ...dp-centos-6.7.json => base-centos-6.7.json} | 17 ++++------------- metron-deployment/packer-build/bin/bento | 8 +++++++- .../packer-build/metron-centos-6.7.json | 5 +++-- metron-deployment/pom.xml | 2 +- .../metron-builder/defaults/main.yml} | 12 +----------- .../roles/metron-builder/tasks/main.yml | 4 +--- .../roles/metron-rpms/defaults/main.yml | 18 ++++++++++++++++++ .../roles/metron-rpms/tasks/main.yml | 2 +- 14 files changed, 49 insertions(+), 48 deletions(-) rename metron-deployment/packer-build/{hdp-centos-6.7.json => base-centos-6.7.json} (88%) rename metron-deployment/{packer-build/ansible/tasks/stop_storm.yml => roles/metron-builder/defaults/main.yml} (69%) create mode 100644 metron-deployment/roles/metron-rpms/defaults/main.yml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index 4523198949..aeb87647c4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -56,6 +56,9 @@ def set_kafka_configured(self): owner=self.__params.metron_user, mode=0775) + def is_hbase_configured(self): + return self.__hbase_configured + def set_hbase_configured(self): Logger.info("Setting HBase Configured to True") File(self.__params.enrichment_hbase_configured_flag_file, @@ -63,6 +66,9 @@ def set_hbase_configured(self): owner=self.__params.metron_user, mode=0775) + def is_geo_configured(self): + return self.__geo_configured + def set_geo_configured(self): Logger.info("Setting GEO Configured to True") File(self.__params.enrichment_geo_configured_flag_file, diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index cced92c709..362b2e2b09 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -50,9 +50,11 @@ def start(self, env, upgrade_type=None): commands = EnrichmentCommands(params) metron_service.load_global_config(params) - if not commands.is_configured(): + if not commands.is_kafka_configured(): commands.init_kafka_topics() + if not commands.is_hbase_configured(): commands.create_hbase_tables() + if not commands.is_geo_configured(): commands.init_geo() commands.start_enrichment_topology() diff --git a/metron-deployment/packer-build/ansible/playbook.yml b/metron-deployment/packer-build/ansible/playbook.yml index 8f922baec3..636b7e584c 100644 --- a/metron-deployment/packer-build/ansible/playbook.yml +++ b/metron-deployment/packer-build/ansible/playbook.yml @@ -26,11 +26,3 @@ tags: - packer-cleanup -- hosts: packer - become: true - roles: - - role: ambari_gather_facts - - role: metron_hbase_tables - - role: metron_kafka_topics - tags: - - packer-cleanup diff --git a/metron-deployment/packer-build/ansible/tasks/cleanup.yml b/metron-deployment/packer-build/ansible/tasks/cleanup.yml index 4c321c9b26..cdeab42989 100644 --- a/metron-deployment/packer-build/ansible/tasks/cleanup.yml +++ b/metron-deployment/packer-build/ansible/tasks/cleanup.yml @@ -15,8 +15,6 @@ # limitations under the License. # --- -- include: ../../roles/ambari_config/tasks/start_hdp.yml - - name: Stop Monit service: name: monit @@ -24,10 +22,6 @@ register: command_result failed_when: "command_result|failed and 'no service or tool found for: monit' not in command_result.msg" -- include: stop_storm.yml - -- include: clear_topics.yml - - name: Delete ES Indices uri: method: DELETE @@ -45,5 +39,4 @@ - include: remove_services.yml -- include: ../../roles/ambari_config/tasks/start_hdp.yml diff --git a/metron-deployment/packer-build/ansible/tasks/clear_topics.yml b/metron-deployment/packer-build/ansible/tasks/clear_topics.yml index cc39419897..11791d33d1 100644 --- a/metron-deployment/packer-build/ansible/tasks/clear_topics.yml +++ b/metron-deployment/packer-build/ansible/tasks/clear_topics.yml @@ -15,6 +15,7 @@ # limitations under the License. # --- +#TODO: Delete the control files too - name: "Delete topics" shell: "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --delete --topic {{ item }}" register: result diff --git a/metron-deployment/packer-build/ansible/tasks/remove_services.yml b/metron-deployment/packer-build/ansible/tasks/remove_services.yml index 9d2477c1ea..922eeea45c 100644 --- a/metron-deployment/packer-build/ansible/tasks/remove_services.yml +++ b/metron-deployment/packer-build/ansible/tasks/remove_services.yml @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +#TODO - use the new module +#TODO - consider removing the services from the blueprint. --- - name: Remove Secondary Namenode uri: diff --git a/metron-deployment/packer-build/hdp-centos-6.7.json b/metron-deployment/packer-build/base-centos-6.7.json similarity index 88% rename from metron-deployment/packer-build/hdp-centos-6.7.json rename to metron-deployment/packer-build/base-centos-6.7.json index ec28b1af2c..92eda67039 100644 --- a/metron-deployment/packer-build/hdp-centos-6.7.json +++ b/metron-deployment/packer-build/base-centos-6.7.json @@ -82,18 +82,9 @@ ], "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'" }, - { - "type" : "ansible", - "pause_before" : "2m", - "playbook_file" : "./ansible/playbook.yml", - "sftp_command": "/usr/libexec/openssh/sftp-server", - "user" : "vagrant", - "host_alias" : "node1", - "groups" : ["packer","ambari_master","ambari_slave"], - "extra_arguments": [ "--tags", "ambari", "--skip-tags","packer-cleanup" ] - }, { "type": "shell", + "pause_before" : "2m", "scripts": [ "scripts/common/vmtools.sh", "scripts/centos/cleanup.sh", @@ -109,7 +100,7 @@ } ], "variables": { - "box_basename": "hdp-centos-6.7", + "box_basename": "base-centos-6.7", "build_timestamp": "{{isotime \"20060102150405\"}}", "cpus": "1", "git_revision": "__unknown_git_revision__", @@ -124,9 +115,9 @@ "metadata": "floppy/dummy_metadata.json", "mirror": "http://archive.kernel.org/centos-vault", "mirror_directory": "6.7/isos/x86_64", - "name": "hdp-centos-6.7", + "name": "base-centos-6.7", "no_proxy": "{{env `no_proxy`}}", - "template": "hdp-centos-6.7", + "template": "base-centos-6.7", "version": "2.1.TIMESTAMP" } } diff --git a/metron-deployment/packer-build/bin/bento b/metron-deployment/packer-build/bin/bento index 982fc8d090..5594345df0 100755 --- a/metron-deployment/packer-build/bin/bento +++ b/metron-deployment/packer-build/bin/bento @@ -66,6 +66,10 @@ class Options options.debug = opt end + opts.on("-a", "--ask", "Run packer with on-error=ask") do |opt| + options.ask = opt + end + opts.on("-o BUILDS", "--only BUILDS", "Only build some Packer builds") do |opt| options.builds = opt end @@ -186,12 +190,13 @@ class BuildRunner include Common include PackerExec - attr_reader :templates, :dry_run, :debug, :builds, :except, :mirror, :override_version, :build_timestamp + attr_reader :templates, :dry_run, :debug, :ask, :builds, :except, :mirror, :override_version, :build_timestamp def initialize(opts) @templates = opts.templates @dry_run = opts.dry_run @debug = opts.debug + @ask = opts.ask @builds = opts.builds @except = opts.except @mirror = opts.mirror @@ -234,6 +239,7 @@ class BuildRunner cmd.insert(2, "headless=true") if headless cmd.insert(2, "-var") if headless cmd.insert(2, "-debug") if debug + cmd.insert(2, "-on-error=ask") if ask cmd.insert(0, "echo") if dry_run cmd end diff --git a/metron-deployment/packer-build/metron-centos-6.7.json b/metron-deployment/packer-build/metron-centos-6.7.json index a4cf625917..19762f772e 100644 --- a/metron-deployment/packer-build/metron-centos-6.7.json +++ b/metron-deployment/packer-build/metron-centos-6.7.json @@ -89,7 +89,8 @@ "sftp_command": "/usr/libexec/openssh/sftp-server", "user" : "vagrant", "host_alias" : "node1", - "groups" : ["packer","ambari_master","ambari_slave","metron_kafka_topics","metron_hbase_tables","enrichment","search","web","sensors","pcap_server","metron"] + "groups" : ["packer","ambari_master","ambari_slave","metron","search","web","sensors","pcap_server","zeppelin"], + "extra_arguments": [ "--extra-vars","metron_rpm_glob=../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm metron_build_dir=../..","--skip-tags","build,sensors,quick_dev" ] }, { "type": "shell", @@ -121,7 +122,7 @@ "ks_path": "centos-6.7/ks.cfg", "memory": "8192", "metadata": "floppy/dummy_metadata.json", - "mirror": "http://mirrors.kernel.org/centos", + "mirror": "http://archive.kernel.org/centos-vault", "mirror_directory": "6.7/isos/x86_64", "name": "metron-centos-6.7", "no_proxy": "{{env `no_proxy`}}", diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml index 0d7e9bb9d9..1179906ba3 100644 --- a/metron-deployment/pom.xml +++ b/metron-deployment/pom.xml @@ -37,7 +37,7 @@ - default + mpack true diff --git a/metron-deployment/packer-build/ansible/tasks/stop_storm.yml b/metron-deployment/roles/metron-builder/defaults/main.yml similarity index 69% rename from metron-deployment/packer-build/ansible/tasks/stop_storm.yml rename to metron-deployment/roles/metron-builder/defaults/main.yml index 1c1b647a90..07f22f4099 100644 --- a/metron-deployment/packer-build/ansible/tasks/stop_storm.yml +++ b/metron-deployment/roles/metron-builder/defaults/main.yml @@ -15,14 +15,4 @@ # limitations under the License. # --- -- name: "List Storm Topologies" - shell: storm list | grep ACTIVE | awk '{print $1}' - register: topologies - -- name: "Kill Storm Topologies" - shell: "storm kill {{ item }}" - register: result - with_items: - - "{{ topologies.stdout_lines }}" - failed_when: result.rc != 0 and ("NotAliveException" not in result.stdout) - changed_when: result.rc == 0 +metron_build_dir: "{{ playbook_dir }}/../.." diff --git a/metron-deployment/roles/metron-builder/tasks/main.yml b/metron-deployment/roles/metron-builder/tasks/main.yml index b028524dca..889eafea84 100644 --- a/metron-deployment/roles/metron-builder/tasks/main.yml +++ b/metron-deployment/roles/metron-builder/tasks/main.yml @@ -16,8 +16,6 @@ # --- - name: Build Deployment Artifacts - local_action: shell cd {{ playbook_dir }}/../.. && mvn clean package -DskipTests -P HDP-2.5.0.0,build-rpms + local_action: shell cd {{ metron_build_dir }} && mvn clean package -DskipTests -P HDP-2.5.0.0,mpack,build-rpms become: false run_once: true - - diff --git a/metron-deployment/roles/metron-rpms/defaults/main.yml b/metron-deployment/roles/metron-rpms/defaults/main.yml new file mode 100644 index 0000000000..4a9d606ec4 --- /dev/null +++ b/metron-deployment/roles/metron-rpms/defaults/main.yml @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +metron_rpm_glob: "{{ playbook_dir }}/../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm" diff --git a/metron-deployment/roles/metron-rpms/tasks/main.yml b/metron-deployment/roles/metron-rpms/tasks/main.yml index c22487cce6..265e835000 100644 --- a/metron-deployment/roles/metron-rpms/tasks/main.yml +++ b/metron-deployment/roles/metron-rpms/tasks/main.yml @@ -29,4 +29,4 @@ owner: root mode: 0755 with_fileglob: - - "{{ playbook_dir }}/../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm" \ No newline at end of file + - "{{ metron_rpm_glob }}" From 33abe8cf13c347a2dfdece145a7b8c17f2a423c0 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Fri, 3 Mar 2017 08:43:07 -0500 Subject: [PATCH 07/17] Fixes from testing Quick Dev --- ...tos-6.7.json => quick-dev-centos-6.7.json} | 10 +++++----- .../playbooks/metron_install.yml | 15 ++++++++++----- .../roles/ambari_slave/tasks/main.yml | 5 ++++- .../roles/quick_dev/meta/main.yml | 19 +++++++++++++++++++ .../roles/quick_dev/tasks/main.yml | 10 ++++++++++ .../vagrant/quick-dev-platform/Vagrantfile | 8 +++++--- 6 files changed, 53 insertions(+), 14 deletions(-) rename metron-deployment/packer-build/{metron-centos-6.7.json => quick-dev-centos-6.7.json} (94%) create mode 100644 metron-deployment/roles/quick_dev/meta/main.yml diff --git a/metron-deployment/packer-build/metron-centos-6.7.json b/metron-deployment/packer-build/quick-dev-centos-6.7.json similarity index 94% rename from metron-deployment/packer-build/metron-centos-6.7.json rename to metron-deployment/packer-build/quick-dev-centos-6.7.json index 19762f772e..f824f055af 100644 --- a/metron-deployment/packer-build/metron-centos-6.7.json +++ b/metron-deployment/packer-build/quick-dev-centos-6.7.json @@ -89,8 +89,8 @@ "sftp_command": "/usr/libexec/openssh/sftp-server", "user" : "vagrant", "host_alias" : "node1", - "groups" : ["packer","ambari_master","ambari_slave","metron","search","web","sensors","pcap_server","zeppelin"], - "extra_arguments": [ "--extra-vars","metron_rpm_glob=../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm metron_build_dir=../..","--skip-tags","build,sensors,quick_dev" ] + "groups" : ["packer","ambari_master","ambari_slave","metron","search","web","sensors","pcap_server","zeppelin","monit"], + "extra_arguments": [ "--extra-vars","metron_rpm_glob=../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm metron_build_dir=../..","--skip-tags","sensors,quick_dev" ] }, { "type": "shell", @@ -109,7 +109,7 @@ } ], "variables": { - "box_basename": "metron-centos-6.7", + "box_basename": "quick-dev-centos-6.7", "build_timestamp": "{{isotime \"20060102150405\"}}", "cpus": "1", "git_revision": "__unknown_git_revision__", @@ -124,9 +124,9 @@ "metadata": "floppy/dummy_metadata.json", "mirror": "http://archive.kernel.org/centos-vault", "mirror_directory": "6.7/isos/x86_64", - "name": "metron-centos-6.7", + "name": "quick-dev-centos-6.7", "no_proxy": "{{env `no_proxy`}}", - "template": "metron-centos-6.7", + "template": "quick-dev-centos-6.7", "version": "2.1.TIMESTAMP" } } diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml index f8f9bf017a..6d31800f21 100644 --- a/metron-deployment/playbooks/metron_install.yml +++ b/metron-deployment/playbooks/metron_install.yml @@ -15,6 +15,16 @@ # limitations under the License. # --- +- hosts: metron + become: true + roles: + - role: ambari_slave + - role: metron-builder + - role: metron-rpms + - role: quick_dev + tags: + - quick_dev + - - hosts: ec2 become: true tasks: @@ -82,8 +92,3 @@ become: false roles: - { role: deployment-report, tags: ['report'] } - -- hosts: metron - become: true - roles: - - { role: quick_dev, tags: ['quick_dev']} \ No newline at end of file diff --git a/metron-deployment/roles/ambari_slave/tasks/main.yml b/metron-deployment/roles/ambari_slave/tasks/main.yml index ab966db024..3fcd24fde2 100644 --- a/metron-deployment/roles/ambari_slave/tasks/main.yml +++ b/metron-deployment/roles/ambari_slave/tasks/main.yml @@ -45,7 +45,10 @@ - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'} - name: Ensure ambari-agent is running - service: name=ambari-agent state=restarted + service: + name: ambari-agent + state: restarted + enabled: yes - name : Wait for agent to register command : sleep 10 diff --git a/metron-deployment/roles/quick_dev/meta/main.yml b/metron-deployment/roles/quick_dev/meta/main.yml new file mode 100644 index 0000000000..c3d807b119 --- /dev/null +++ b/metron-deployment/roles/quick_dev/meta/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - ambari_gather_facts diff --git a/metron-deployment/roles/quick_dev/tasks/main.yml b/metron-deployment/roles/quick_dev/tasks/main.yml index 9825e71adc..cd347911b1 100644 --- a/metron-deployment/roles/quick_dev/tasks/main.yml +++ b/metron-deployment/roles/quick_dev/tasks/main.yml @@ -15,6 +15,16 @@ # limitations under the License. # --- +- name: Start the ambari cluster + ambari_cluster_state: + host: "{{ groups.ambari_master[0] }}" + port: "{{ ambari_port }}" + username: "{{ ambari_user }}" + password: "{{ ambari_password }}" + cluster_name: "{{ cluster_name }}" + cluster_state: started + wait_for_complete: True + - name: Shut Down Metron ambari_service_state: host: "{{ groups.ambari_master[0] }}" diff --git a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile index 28c73da9c9..dd4920e428 100644 --- a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile +++ b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile @@ -16,8 +16,8 @@ # require 'getoptlong' -ansibleTags='ambari-agent,hdp-deploy,metron' -ansibleSkipTags='solr,sensors' +ansibleTags='quick_dev' +ansibleSkipTags='' begin opts = GetoptLong.new( @@ -53,7 +53,8 @@ hosts = [{ Vagrant.configure(2) do |config| # all hosts built on centos 6 - config.vm.box = "metron/hdp-base" + config.vm.box_url = "file:///Users/dlyle/projects/metron-dlyle/metron-deployment/packer-build/builds/quick-dev-centos-6.7-2.1.20170302201505.git.c49b2af13ff4f061dfdb5620f01ef5b3337c612b_dirty.virtualbox.box" + config.vm.box = "test-hdp-box" config.ssh.insert_key = true # enable the hostmanager plugin @@ -88,6 +89,7 @@ Vagrant.configure(2) do |config| ansible.tags = ansibleTags.split(",") if ansibleTags != '' ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != '' ansible.inventory_path = "../../inventory/quick-dev-platform" + ansible.verbose = true end end From cd4bed26de04b196c5331f9c095e34d0a3e11a93 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Fri, 3 Mar 2017 18:44:04 -0500 Subject: [PATCH 08/17] Working quick dev and full dev from local box files. --- metron-deployment/packer-build/quick-dev-centos-6.7.json | 2 +- metron-deployment/vagrant/full-dev-platform/Vagrantfile | 3 ++- metron-deployment/vagrant/quick-dev-platform/Vagrantfile | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/metron-deployment/packer-build/quick-dev-centos-6.7.json b/metron-deployment/packer-build/quick-dev-centos-6.7.json index f824f055af..919c4b3cd5 100644 --- a/metron-deployment/packer-build/quick-dev-centos-6.7.json +++ b/metron-deployment/packer-build/quick-dev-centos-6.7.json @@ -90,7 +90,7 @@ "user" : "vagrant", "host_alias" : "node1", "groups" : ["packer","ambari_master","ambari_slave","metron","search","web","sensors","pcap_server","zeppelin","monit"], - "extra_arguments": [ "--extra-vars","metron_rpm_glob=../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm metron_build_dir=../..","--skip-tags","sensors,quick_dev" ] + "extra_arguments": [ "--extra-vars","metron_rpm_glob=../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm metron_build_dir=../..","--skip-tags","report,sensors,quick_dev" ] }, { "type": "shell", diff --git a/metron-deployment/vagrant/full-dev-platform/Vagrantfile b/metron-deployment/vagrant/full-dev-platform/Vagrantfile index 47165167ae..24615acdd9 100644 --- a/metron-deployment/vagrant/full-dev-platform/Vagrantfile +++ b/metron-deployment/vagrant/full-dev-platform/Vagrantfile @@ -53,7 +53,8 @@ hosts = [{ Vagrant.configure(2) do |config| # all hosts built on centos 6 - config.vm.box = "bento/centos-6.7" + config.vm.box = "new_base" + config.vm.box_url = "/Users/dml/projects/metron-dlyle/metron-deployment/packer-build/builds/base-centos-6.7-2.1.20170303223924.git.33abe8cf13c347a2dfdece145a7b8c17f2a423c0_dirty.virtualbox.box" config.ssh.insert_key = true # enable the hostmanager plugin diff --git a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile index dd4920e428..ada8a4eedf 100644 --- a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile +++ b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile @@ -16,7 +16,7 @@ # require 'getoptlong' -ansibleTags='quick_dev' +ansibleTags='quick_dev,report' ansibleSkipTags='' begin @@ -53,7 +53,7 @@ hosts = [{ Vagrant.configure(2) do |config| # all hosts built on centos 6 - config.vm.box_url = "file:///Users/dlyle/projects/metron-dlyle/metron-deployment/packer-build/builds/quick-dev-centos-6.7-2.1.20170302201505.git.c49b2af13ff4f061dfdb5620f01ef5b3337c612b_dirty.virtualbox.box" + config.vm.box_url = "file:///Users/dml/projects/metron-dlyle/metron-deployment/packer-build/builds/quick-dev-centos-6.7-2.1.20170303191109.git.33abe8cf13c347a2dfdece145a7b8c17f2a423c0_dirty.virtualbox.box" config.vm.box = "test-hdp-box" config.ssh.insert_key = true From 3e81f685f5c323698eaf8f80ecf9a4cad3f5f092 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Sat, 4 Mar 2017 13:49:09 -0500 Subject: [PATCH 09/17] Working with remote boxes. --- .../inventory/metron_example/group_vars/all | 93 ------------------- .../inventory/metron_example/hosts | 52 ----------- .../ansible/tasks/clear_topics.yml | 30 ------ .../ansible/tasks/remove_services.yml | 3 - .../roles/bro/tasks/bro-plugin-kafka.yml | 2 +- .../vagrant/full-dev-platform/Vagrantfile | 3 +- .../vagrant/quick-dev-platform/Vagrantfile | 5 +- 7 files changed, 4 insertions(+), 184 deletions(-) delete mode 100644 metron-deployment/inventory/metron_example/group_vars/all delete mode 100644 metron-deployment/inventory/metron_example/hosts delete mode 100644 metron-deployment/packer-build/ansible/tasks/clear_topics.yml diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all deleted file mode 100644 index 614925df50..0000000000 --- a/metron-deployment/inventory/metron_example/group_vars/all +++ /dev/null @@ -1,93 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# which services should be started? -services_to_start: - - pcap-service - - yaf - - snort - - snort-logs - - bro - - pcap-replay - -#Ansible Variables -ansible_ssh_private_key_file: /Path/to/private/key/file #Change This -ansible_ssh_user: root - -#Ambari -ambari_host: "{{ groups.ambari_master[0] }}" -ambari_port: 8080 -ambari_user: admin -ambari_password: admin -cluster_type: small_cluster - -# hbase -pcap_hbase_table: pcap -tracker_hbase_table: access_tracker -threatintel_hbase_table: threatintel -enrichment_hbase_table: enrichment - -# kafka -num_partitions: 3 -retention_in_gb: 25 - -# metron variables -metron_version: 0.3.1 -metron_directory: /usr/metron/{{ metron_version }} -pcapservice_port: 8081 - -# sensors -sniff_interface: eth1 -bro_version: "2.4.1" -fixbuf_version: "1.7.1" -yaf_version: "2.8.0" -daq_version: "2.0.6-1" -iface: "eth0" -pycapa_repo: "https://github.com/OpenSOC/pycapa.git" -pycapa_home: "/opt/pycapa" -snort_version: "2.9.8.0-1" -snort_alert_csv_path: "/var/log/snort/alert.csv" - -# pcap-replay -install_pcap_replay: True -install_tap: True -tap_ip: 10.0.0.1 -sensor_test_mode: True -pcap_replay_interface: tap0 -pcap_replay_home: /opt/pcap-replay - -# data directories -zookeeper_data_dir: "/data1/hadoop/zookeeper" -namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary" -namenode_name_dir: "/data1/hadoop/hdfs/namenode" -datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data" -journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode" -nodemanager_local_dirs: "/data1/hadoop/yarn/local" -timeline_ldb_store_path: "/data1/hadoop/yarn/timeline" -timeline_ldb_state_path: "/data1/hadoop/yarn/timeline" -nodemanager_log_dirs: "/data1/hadoop/yarn/log" -jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs" -storm_local_dir: "/data1/hadoop/storm" -kafka_log_dirs: "/data2/kafka-log" -elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch" - -# search -install_elasticsearch: True -install_solr: False -elasticsearch_transport_port: 9300 -elasticsearch_network_interface: eth0 -elasticsearch_web_port: 9200 diff --git a/metron-deployment/inventory/metron_example/hosts b/metron-deployment/inventory/metron_example/hosts deleted file mode 100644 index 8f09e4e462..0000000000 --- a/metron-deployment/inventory/metron_example/hosts +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -[ambari_master] -node1 - -#minimum of 3 - 6 from 12 node cluser -[ambari_slave] -node2 -node3 -node4 -node5 -node6 -node7 -node8 - -#3rd ambari_slave -[metron] -node4 - -#1 or more -[search] -node10 -node11 -node12 - -[sensors] -node1 - -[pcap_server] -node9 - -[web] -node12 - -[monit:children] -sensors -pcap_server diff --git a/metron-deployment/packer-build/ansible/tasks/clear_topics.yml b/metron-deployment/packer-build/ansible/tasks/clear_topics.yml deleted file mode 100644 index 11791d33d1..0000000000 --- a/metron-deployment/packer-build/ansible/tasks/clear_topics.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -#TODO: Delete the control files too -- name: "Delete topics" - shell: "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --delete --topic {{ item }}" - register: result - with_items: - - bro - - yaf - - snort - - enrichments - - pcap - failed_when: result.rc != 0 and ("does not exist" not in result.stderr) - changed_when: result.rc == 0 - diff --git a/metron-deployment/packer-build/ansible/tasks/remove_services.yml b/metron-deployment/packer-build/ansible/tasks/remove_services.yml index 922eeea45c..9d2477c1ea 100644 --- a/metron-deployment/packer-build/ansible/tasks/remove_services.yml +++ b/metron-deployment/packer-build/ansible/tasks/remove_services.yml @@ -14,9 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -#TODO - use the new module -#TODO - consider removing the services from the blueprint. --- - name: Remove Secondary Namenode uri: diff --git a/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml b/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml index ad734985e2..fe57ef28ec 100644 --- a/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml +++ b/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml @@ -41,4 +41,4 @@ - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ kafka_broker_url }}\");" - name: Deploy bro configuration changes - shell: "{{ bro_home }}/bin/broctl deploy" \ No newline at end of file + shell: "{{ bro_home }}/bin/broctl deploy" diff --git a/metron-deployment/vagrant/full-dev-platform/Vagrantfile b/metron-deployment/vagrant/full-dev-platform/Vagrantfile index 24615acdd9..1c96ded6d9 100644 --- a/metron-deployment/vagrant/full-dev-platform/Vagrantfile +++ b/metron-deployment/vagrant/full-dev-platform/Vagrantfile @@ -53,8 +53,7 @@ hosts = [{ Vagrant.configure(2) do |config| # all hosts built on centos 6 - config.vm.box = "new_base" - config.vm.box_url = "/Users/dml/projects/metron-dlyle/metron-deployment/packer-build/builds/base-centos-6.7-2.1.20170303223924.git.33abe8cf13c347a2dfdece145a7b8c17f2a423c0_dirty.virtualbox.box" + config.vm.box = "metron/centos_base" config.ssh.insert_key = true # enable the hostmanager plugin diff --git a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile index ada8a4eedf..fb9c4e285b 100644 --- a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile +++ b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile @@ -16,7 +16,7 @@ # require 'getoptlong' -ansibleTags='quick_dev,report' +ansibleTags='quick_dev' ansibleSkipTags='' begin @@ -53,8 +53,7 @@ hosts = [{ Vagrant.configure(2) do |config| # all hosts built on centos 6 - config.vm.box_url = "file:///Users/dml/projects/metron-dlyle/metron-deployment/packer-build/builds/quick-dev-centos-6.7-2.1.20170303191109.git.33abe8cf13c347a2dfdece145a7b8c17f2a423c0_dirty.virtualbox.box" - config.vm.box = "test-hdp-box" + config.vm.box = "metron/quick_dev" config.ssh.insert_key = true # enable the hostmanager plugin From aabdef22ba2775bb5e55057645002ff0e080da6c Mon Sep 17 00:00:00 2001 From: David Lyle Date: Sat, 4 Mar 2017 14:25:40 -0500 Subject: [PATCH 10/17] Re-order cluster start and service delete to be more efficient. --- .../roles/quick_dev/tasks/main.yml | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/metron-deployment/roles/quick_dev/tasks/main.yml b/metron-deployment/roles/quick_dev/tasks/main.yml index cd347911b1..0fa0a23d3f 100644 --- a/metron-deployment/roles/quick_dev/tasks/main.yml +++ b/metron-deployment/roles/quick_dev/tasks/main.yml @@ -15,27 +15,6 @@ # limitations under the License. # --- -- name: Start the ambari cluster - ambari_cluster_state: - host: "{{ groups.ambari_master[0] }}" - port: "{{ ambari_port }}" - username: "{{ ambari_user }}" - password: "{{ ambari_password }}" - cluster_name: "{{ cluster_name }}" - cluster_state: started - wait_for_complete: True - -- name: Shut Down Metron - ambari_service_state: - host: "{{ groups.ambari_master[0] }}" - port: "{{ ambari_port }}" - username: "{{ ambari_user }}" - password: "{{ ambari_password }}" - cluster_name: "{{ cluster_name }}" - state: stopped - service_name: METRON - wait_for_complete: True - - name: Delete the Metron Components from Ambari ambari_service_state: host: "{{ groups.ambari_master[0] }}" @@ -79,14 +58,13 @@ - METRON_INDEXING - METRON_PARSERS -- name: Start Metron - ambari_service_state: +- name: Start the ambari cluster + ambari_cluster_state: host: "{{ groups.ambari_master[0] }}" port: "{{ ambari_port }}" username: "{{ ambari_user }}" password: "{{ ambari_password }}" cluster_name: "{{ cluster_name }}" - state: started - service_name: METRON + cluster_state: started wait_for_complete: True From ad784c3b9670b31a289725be762c1ab839fb84a2 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Sat, 4 Mar 2017 17:03:10 -0500 Subject: [PATCH 11/17] Modified docs and moved packer-build to packaging. --- metron-deployment/README.md | 98 +++++-------------- .../{ => packaging}/packer-build/.gitignore | 2 +- .../{ => packaging}/packer-build/README.md | 19 ++-- .../{ => packaging}/packer-build/ansible.cfg | 4 +- .../packer-build/ansible/playbook.yml | 2 +- .../packer-build/ansible/tasks/cleanup.yml | 0 .../ansible/tasks/remove_services.yml | 0 .../packer-build/base-centos-6.7.json | 0 .../{ => packaging}/packer-build/bin/bento | 0 .../packer-build/http/centos-6.7/ks.cfg | 0 .../packer-build/quick-dev-centos-6.7.json | 2 +- .../packer-build/scripts/centos/cleanup.sh | 0 .../packer-build/scripts/centos/networking.sh | 0 .../packer-build/scripts/centos/update_all.sh | 0 .../packer-build/scripts/common/metadata.sh | 0 .../packer-build/scripts/common/minimize.sh | 0 .../packer-build/scripts/common/reboot.sh | 0 .../packer-build/scripts/common/sshd.sh | 0 .../packer-build/scripts/common/sudoers.sh | 0 .../packer-build/scripts/common/vagrant.sh | 0 .../packer-build/scripts/common/vmtools.sh | 0 21 files changed, 40 insertions(+), 87 deletions(-) rename metron-deployment/{ => packaging}/packer-build/.gitignore (53%) rename metron-deployment/{ => packaging}/packer-build/README.md (65%) rename metron-deployment/{ => packaging}/packer-build/ansible.cfg (94%) rename metron-deployment/{ => packaging}/packer-build/ansible/playbook.yml (94%) rename metron-deployment/{ => packaging}/packer-build/ansible/tasks/cleanup.yml (100%) rename metron-deployment/{ => packaging}/packer-build/ansible/tasks/remove_services.yml (100%) rename metron-deployment/{ => packaging}/packer-build/base-centos-6.7.json (100%) rename metron-deployment/{ => packaging}/packer-build/bin/bento (100%) rename metron-deployment/{ => packaging}/packer-build/http/centos-6.7/ks.cfg (100%) rename metron-deployment/{ => packaging}/packer-build/quick-dev-centos-6.7.json (95%) rename metron-deployment/{ => packaging}/packer-build/scripts/centos/cleanup.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/centos/networking.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/centos/update_all.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/common/metadata.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/common/minimize.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/common/reboot.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/common/sshd.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/common/sudoers.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/common/vagrant.sh (100%) rename metron-deployment/{ => packaging}/packer-build/scripts/common/vmtools.sh (100%) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index b9733536a0..e5a1ff0b01 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -1,97 +1,51 @@ # Overview -This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster, Metron services, or both using ansible -playbooks. These playbooks currently only target RHEL/CentOS 6.x operating +This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster containing Metron services using Ansible. These playbooks target RHEL/CentOS 6.x operating systems. -In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed in this README. +Installation consists of - +- Building Metron tarballs, RPMs and the Ambari MPack +- Deploying Ambari +- Leveraging Ambari to install: + * The required Hadoop Components + * Core Metron (Parsing, Enrichment, Indexing) + * Elasticsearch + * Kibana +- Starting All Services ## Prerequisites The following tools are required to run these scripts: - [Maven](https://maven.apache.org/) - [Git](https://git-scm.com/) -- [Ansible](http://www.ansible.com/) (version 2.0 or greater) - -Currently Metron must be built from source. Before running these scripts perform the following steps: - -1. Clone the Metron git repository with `git clone git@github.com:apache/incubator-metron.git` -2. Navigate to `incubator-metron` and run `mvn clean package` +- [Ansible](http://www.ansible.com/) (version 2.0.0.2) +- [Docker](https://www.docker.com/) (Docker for Mac on OSX) These scripts depend on two files for configuration: - hosts - declares which Ansible roles will be run on which hosts - group_vars/all - various configuration settings needed to install Metron -Examples can be found in the -`incubator-metron/metron-deployment/inventory/metron_example` directory and are a good starting point. Copy this directory -into `incubator-metron/metron-deployment/inventory/` and rename it to your `project_name`. More information about Ansible files and directory -structure can be found at http://docs.ansible.com/ansible/playbooks_best_practices.html. - +For production use, it is recommended that Metron be installed on an existing cluster managed by Ambari as described in the Installing Management Pack section below. ## Ambari -The Ambari playbook will install a Hadoop cluster with all the services and configuration required by Metron. This -section can be skipped if installing Metron on a pre-existing cluster. - -Currently, this playbook supports building a local development cluster running on one node but options for other types - of clusters will be added in the future. - -### Setting up your inventory -Make sure to update the hosts file in `incubator-metron/metron-deployment/inventory/project_name/hosts` or provide an -alternate inventory file when you launch the playbooks, including the -ssh user(s) and ssh keyfile location(s). These playbooks expect two -host groups: - -- ambari_master -- ambari_slaves - -### Running the playbook -This playbook will install the Ambari server on the ambari_master, install the ambari agents on -the ambari_slaves, and create a cluster in Ambari with a blueprint for the required -Metron components. - -Navigate to `incubator-metron/metron-deployment/playbooks` and run: -`ansible-playbook -i ../inventory/project_name ambari_install.yml` - -## Metron -The Metron playbook will gather the necessary cluster settings from Ambari and install the Metron services. +The Ambari playbook will install a Hadoop cluster including the Metron Services (Parsing, Enrichment, Indexing). Ambari will also install Elasticsearch and Kibana. -### Setting up your inventory -Edit the hosts file at `incubator-metron/metron-deployment/inventory/project_name/hosts`. Declare where which hosts the -Metron services will be installed on by updating these groups: - -- enrichment - submits the topology code to Storm and requires a storm client -- search - host where Elasticsearch will be run -- web - host where the Metron UI and underlying services will run -- sensors - host where network data will be collected and published to Kafka - -The Metron topologies depend on Kafka topics and HBase tables being created beforehand. Declare a host that has Kafka and HBase clients installed by updating these groups: - -- metron_kafka_topics -- metron_hbase_tables - -If only installing Metron, these groups can be ignored: - -- ambari_master -- ambari_slaves - -### Configuring group variables -The Metron Ansible scripts depend on a set of variables. These variables can be found in the file at -`incubator-metron/metron-deployment/inventory/project_name/group_vars/all`. Edit the ambari* variables to match your Ambari -instance and update the java_home variable to match the java path on your hosts. - -### Running the playbook -Navigate to `incubator-metron/metron-deployment/playbooks` and run: -`ansible-playbook -i ../inventory/project_name metron_install.yml` +Currently, the playbooks supports building a local development cluster running on one node or deploying to a 10 node cluster on AWS EC2. ## Vagrant -A VagrantFile is included and will install a working version of the entire Metron stack. The following is required to -run this: +There are current two Vagrant modes, full-dev and quick-dev. Full-dev installs the entire Ambari/Metron stack. This is useful in testing out changes to the installation procedure. +Quick-dev re-installs the core Metron Services (Parsing, Enrichment, and Indexing)on a pre-built instance. +Use quick-dev for testing out changes to core Metron services. -- [Vagrant](https://www.vagrantup.com/) -- Hostmanager plugin for vagrant - Run `vagrant plugin install vagrant-hostmanager` on the machine where Vagrant is +### Prerequsities +- Install [Vagrant](https://www.vagrantup.com/) (5.0.16+) +- Install the Hostmanager plugin for vagrant - Run `vagrant plugin install vagrant-hostmanager` on the machine where Vagrant is installed -Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`. This also provides a good -example of how to run a full end-to-end Metron install. +### Full-Dev +Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`. + +### Quick-Dev +Navigate to `incubator-metron/metron-deployment/vagrant/quick-dev-platform` and run `vagrant up`. ## Ambari Management Pack An Ambari Management Pack can be built in order to make the Metron service available on top of an existing stack, rather than needing a direct stack update. diff --git a/metron-deployment/packer-build/.gitignore b/metron-deployment/packaging/packer-build/.gitignore similarity index 53% rename from metron-deployment/packer-build/.gitignore rename to metron-deployment/packaging/packer-build/.gitignore index 68bf87b990..1b72634b67 100644 --- a/metron-deployment/packer-build/.gitignore +++ b/metron-deployment/packaging/packer-build/.gitignore @@ -1,6 +1,6 @@ .bundle/ iso -*.box +builds/base-centos-6.7-2.1.20170303223924.git.33abe8cf13c347a2dfdece145a7b8c17f2a423c0_dirty.virtualbox.box virtualfloppy.vfd packer_cache packer.log diff --git a/metron-deployment/packer-build/README.md b/metron-deployment/packaging/packer-build/README.md similarity index 65% rename from metron-deployment/packer-build/README.md rename to metron-deployment/packaging/packer-build/README.md index 1100d5fa3a..d0c54846a3 100644 --- a/metron-deployment/packer-build/README.md +++ b/metron-deployment/packaging/packer-build/README.md @@ -5,14 +5,13 @@ Based on the fantastic [Bento](https://github.com/chef/bento) project developed Images Provided --------------------- -- hdp-centos-6.7: Centos 6.7 + HDP. Used in the quick-dev-platform Vagrant image -- metron-centos-6.7: Centos 6.7 + HDP + Metron. Used for the codelab-platform Vagrant image. +- base-centos-6.7: Centos 6.7 + HDP. Used in the full-dev-platform Vagrant image +- quick-dev-centos-6.7: Centos 6.7 + HDP + Metron. Used for the quick-dev-platform Vagrant image. Prerequisites --------------------- -- [Packer](https://www.packer.io/) 0.10.1 -- [Virtualbox](https://www.virtualbox.org/) 5.0.16 -- Be sure to build Metron prior to building the images (cd *your-project-directory*/metron-platform && mvn clean package -DskipTests) +- [Packer](https://www.packer.io/) 0.12.2 +- [Virtualbox](https://www.virtualbox.org/) 5.0.16+ (Tested with 5.0.20) Build Both Images ---------------------- @@ -24,18 +23,18 @@ Build Both Images Build Single Images ---------------------- Navigate to *your-project-directory*/metron-deployment/packer-build - * HDP Centos + * Base Centos (full-dev) ``` -bin/bento build hdp-centos-6.7.json +bin/bento build base-centos-6.7.json ``` - * Full Metron + * Quick Dev ``` -bin/bento build metron-centos-6.7.json +bin/bento build quick-dev-centos-6.7.json ``` Using Your New Box File ---------------------- -Modify the relevant Vagrantfile (codelab-platform or quick-dev-platform) replacing the lines: +Modify the relevant Vagrantfile (full-dev-platform or quick-dev-platform) replacing the lines: ``` 
config.vm.box = "box_name"
 config.ssh.insert_key = true
diff --git a/metron-deployment/packer-build/ansible.cfg b/metron-deployment/packaging/packer-build/ansible.cfg similarity index 94% rename from metron-deployment/packer-build/ansible.cfg rename to metron-deployment/packaging/packer-build/ansible.cfg index e329517229..eeb5084942 100644 --- a/metron-deployment/packer-build/ansible.cfg +++ b/metron-deployment/packaging/packer-build/ansible.cfg @@ -16,8 +16,8 @@ # [defaults] host_key_checking = false -library = ../extra_modules -roles_path = ../roles +library = ../../extra_modules +roles_path = ../../roles pipelining = True remote_user = vagrant log_path = ./ansible.log diff --git a/metron-deployment/packer-build/ansible/playbook.yml b/metron-deployment/packaging/packer-build/ansible/playbook.yml similarity index 94% rename from metron-deployment/packer-build/ansible/playbook.yml rename to metron-deployment/packaging/packer-build/ansible/playbook.yml index 636b7e584c..0aba491174 100644 --- a/metron-deployment/packer-build/ansible/playbook.yml +++ b/metron-deployment/packaging/packer-build/ansible/playbook.yml @@ -15,7 +15,7 @@ # limitations under the License. # --- -- include: ../../playbooks/metron_full_install.yml +- include: ../../../playbooks/metron_full_install.yml - hosts: packer become: true diff --git a/metron-deployment/packer-build/ansible/tasks/cleanup.yml b/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml similarity index 100% rename from metron-deployment/packer-build/ansible/tasks/cleanup.yml rename to metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml diff --git a/metron-deployment/packer-build/ansible/tasks/remove_services.yml b/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml similarity index 100% rename from metron-deployment/packer-build/ansible/tasks/remove_services.yml rename to metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml diff --git a/metron-deployment/packer-build/base-centos-6.7.json b/metron-deployment/packaging/packer-build/base-centos-6.7.json similarity index 100% rename from metron-deployment/packer-build/base-centos-6.7.json rename to metron-deployment/packaging/packer-build/base-centos-6.7.json diff --git a/metron-deployment/packer-build/bin/bento b/metron-deployment/packaging/packer-build/bin/bento similarity index 100% rename from metron-deployment/packer-build/bin/bento rename to metron-deployment/packaging/packer-build/bin/bento diff --git a/metron-deployment/packer-build/http/centos-6.7/ks.cfg b/metron-deployment/packaging/packer-build/http/centos-6.7/ks.cfg similarity index 100% rename from metron-deployment/packer-build/http/centos-6.7/ks.cfg rename to metron-deployment/packaging/packer-build/http/centos-6.7/ks.cfg diff --git a/metron-deployment/packer-build/quick-dev-centos-6.7.json b/metron-deployment/packaging/packer-build/quick-dev-centos-6.7.json similarity index 95% rename from metron-deployment/packer-build/quick-dev-centos-6.7.json rename to metron-deployment/packaging/packer-build/quick-dev-centos-6.7.json index 919c4b3cd5..96680cbedf 100644 --- a/metron-deployment/packer-build/quick-dev-centos-6.7.json +++ b/metron-deployment/packaging/packer-build/quick-dev-centos-6.7.json @@ -90,7 +90,7 @@ "user" : "vagrant", "host_alias" : "node1", "groups" : ["packer","ambari_master","ambari_slave","metron","search","web","sensors","pcap_server","zeppelin","monit"], - "extra_arguments": [ "--extra-vars","metron_rpm_glob=../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm metron_build_dir=../..","--skip-tags","report,sensors,quick_dev" ] + "extra_arguments": [ "--extra-vars","metron_rpm_glob=../../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm metron_build_dir=../../..","--skip-tags","report,sensors,quick_dev" ] }, { "type": "shell", diff --git a/metron-deployment/packer-build/scripts/centos/cleanup.sh b/metron-deployment/packaging/packer-build/scripts/centos/cleanup.sh similarity index 100% rename from metron-deployment/packer-build/scripts/centos/cleanup.sh rename to metron-deployment/packaging/packer-build/scripts/centos/cleanup.sh diff --git a/metron-deployment/packer-build/scripts/centos/networking.sh b/metron-deployment/packaging/packer-build/scripts/centos/networking.sh similarity index 100% rename from metron-deployment/packer-build/scripts/centos/networking.sh rename to metron-deployment/packaging/packer-build/scripts/centos/networking.sh diff --git a/metron-deployment/packer-build/scripts/centos/update_all.sh b/metron-deployment/packaging/packer-build/scripts/centos/update_all.sh similarity index 100% rename from metron-deployment/packer-build/scripts/centos/update_all.sh rename to metron-deployment/packaging/packer-build/scripts/centos/update_all.sh diff --git a/metron-deployment/packer-build/scripts/common/metadata.sh b/metron-deployment/packaging/packer-build/scripts/common/metadata.sh similarity index 100% rename from metron-deployment/packer-build/scripts/common/metadata.sh rename to metron-deployment/packaging/packer-build/scripts/common/metadata.sh diff --git a/metron-deployment/packer-build/scripts/common/minimize.sh b/metron-deployment/packaging/packer-build/scripts/common/minimize.sh similarity index 100% rename from metron-deployment/packer-build/scripts/common/minimize.sh rename to metron-deployment/packaging/packer-build/scripts/common/minimize.sh diff --git a/metron-deployment/packer-build/scripts/common/reboot.sh b/metron-deployment/packaging/packer-build/scripts/common/reboot.sh similarity index 100% rename from metron-deployment/packer-build/scripts/common/reboot.sh rename to metron-deployment/packaging/packer-build/scripts/common/reboot.sh diff --git a/metron-deployment/packer-build/scripts/common/sshd.sh b/metron-deployment/packaging/packer-build/scripts/common/sshd.sh similarity index 100% rename from metron-deployment/packer-build/scripts/common/sshd.sh rename to metron-deployment/packaging/packer-build/scripts/common/sshd.sh diff --git a/metron-deployment/packer-build/scripts/common/sudoers.sh b/metron-deployment/packaging/packer-build/scripts/common/sudoers.sh similarity index 100% rename from metron-deployment/packer-build/scripts/common/sudoers.sh rename to metron-deployment/packaging/packer-build/scripts/common/sudoers.sh diff --git a/metron-deployment/packer-build/scripts/common/vagrant.sh b/metron-deployment/packaging/packer-build/scripts/common/vagrant.sh similarity index 100% rename from metron-deployment/packer-build/scripts/common/vagrant.sh rename to metron-deployment/packaging/packer-build/scripts/common/vagrant.sh diff --git a/metron-deployment/packer-build/scripts/common/vmtools.sh b/metron-deployment/packaging/packer-build/scripts/common/vmtools.sh similarity index 100% rename from metron-deployment/packer-build/scripts/common/vmtools.sh rename to metron-deployment/packaging/packer-build/scripts/common/vmtools.sh From 37ddde3ae2f26e4efebe1c1c3d2e43a4c70f4b0b Mon Sep 17 00:00:00 2001 From: David Lyle Date: Mon, 6 Mar 2017 15:51:47 -0500 Subject: [PATCH 12/17] Update Upgrading document to reflect reliance on Docker. --- Upgrading.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Upgrading.md b/Upgrading.md index 2a2ea50cb4..79c5ad8f92 100644 --- a/Upgrading.md +++ b/Upgrading.md @@ -152,3 +152,11 @@ into a separate function. The consequence is that existing calls to Existing calls to `PROFILE_GET` will need to change from `PROFILE_GET('profile', 'entity', duration, 'durationUnits')` to `PROFILE_GET('profile', 'entity', PROFILE_FIXED(duration, 'durationUnits'))` +## 0.3.1 to PLACEHOLDER + +### [METRON-671: Refactor existing Ansible deployment to use Ambari MPack](https://issues.apache.org/jira/browse/METRON-671) + +#### Description +Since the Ansible Deployment uses the MPack, RPMs must be built prior to deployment. As a result, +[Docker](https://www.docker.com/) is required to perform a Quick-Dev, Full-Dev or Ansible deployment. +This effectively limits the build environment to Docker supported [platforms](https://docs.docker.com/engine/installation/#platform-support-matrix). From a5049a13d63f444f62ff8880fd491f20ab97a306 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 7 Mar 2017 08:27:11 -0500 Subject: [PATCH 13/17] Remove duplicate dependency declarations that came in with the last merge. It breaks with newer versions of Maven. --- metron-platform/metron-common/pom.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index 8d1e183d24..31848aec18 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -52,12 +52,6 @@ ${project.parent.version} test - - org.apache.hadoop - hadoop-auth - ${global_hadoop_version} - test - commons-validator commons-validator @@ -320,12 +314,6 @@ 2.0.2 test - - org.apache.metron - metron-test-utilities - ${project.parent.version} - test - commons-beanutils commons-beanutils From b13083d6a347f9deeae1ab6c7bc3d9e6d7a22488 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 7 Mar 2017 10:49:43 -0500 Subject: [PATCH 14/17] Fixed deployment report on quick dev. --- .../extra_modules/ambari_service_state.py | 2 +- .../inventory/quick-dev-platform/hosts | 23 ++++++++----------- .../roles/ambari_master/defaults/main.yml | 2 +- .../roles/ambari_master/tasks/main.yml | 4 ++-- .../vagrant/quick-dev-platform/Vagrantfile | 2 +- 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/metron-deployment/extra_modules/ambari_service_state.py b/metron-deployment/extra_modules/ambari_service_state.py index e67bcfe406..6dbefa8d11 100644 --- a/metron-deployment/extra_modules/ambari_service_state.py +++ b/metron-deployment/extra_modules/ambari_service_state.py @@ -155,7 +155,7 @@ def main(): host = p.get('host') port = p.get('port') - username = p.get('password') + username = p.get('username') password = p.get('password') cluster_name = p.get('cluster_name') state = p.get('state') diff --git a/metron-deployment/inventory/quick-dev-platform/hosts b/metron-deployment/inventory/quick-dev-platform/hosts index 98e8681ea9..9bd9ea1e05 100644 --- a/metron-deployment/inventory/quick-dev-platform/hosts +++ b/metron-deployment/inventory/quick-dev-platform/hosts @@ -21,32 +21,27 @@ node1 [ambari_slave] node1 -[metron_hbase_tables] +[metron] node1 -[metron_kafka_topics] +[search] node1 -[enrichment] +[sensors] node1 -[search] +[pcap_server] node1 [web] node1 -[sensors] +[zeppelin] node1 -[pcap_server] -node1 - -[metron:children] -enrichment -search -web +[monit:children] sensors -metron_kafka_topics -metron_hbase_tables pcap_server + +[local] +127.0.0.1 diff --git a/metron-deployment/roles/ambari_master/defaults/main.yml b/metron-deployment/roles/ambari_master/defaults/main.yml index 3b8cc733ee..b043f065db 100644 --- a/metron-deployment/roles/ambari_master/defaults/main.yml +++ b/metron-deployment/roles/ambari_master/defaults/main.yml @@ -16,4 +16,4 @@ # --- ambari_server_mem: 2048 - +ambari_mpack_version: 0.3.1.0 \ No newline at end of file diff --git a/metron-deployment/roles/ambari_master/tasks/main.yml b/metron-deployment/roles/ambari_master/tasks/main.yml index ee185c6094..5b058bdf9a 100644 --- a/metron-deployment/roles/ambari_master/tasks/main.yml +++ b/metron-deployment/roles/ambari_master/tasks/main.yml @@ -40,13 +40,13 @@ - name: Copy MPack to Ambari Host copy: - src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-0.3.1.0.tar.gz" + src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-{{ ambari_mpack_version }}.tar.gz" dest: /tmp - name: Install MPack on Ambari Host shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-0.3.1.0.tar.gz args: - creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-0.3.1.0/addon-services + creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services - name: start ambari server service: diff --git a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile index 2748a74185..1c18df542d 100644 --- a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile +++ b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile @@ -16,7 +16,7 @@ # require 'getoptlong' -ansibleTags='quick_dev' +ansibleTags='quick_dev,report' ansibleSkipTags='' begin From 7a69c992b45d6baf37f6796525acdc5f6ac98750 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Tue, 7 Mar 2017 20:55:02 -0500 Subject: [PATCH 15/17] Replace "duplicate" dependencies that weren't actually duplicated. --- metron-platform/metron-common/pom.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index 31848aec18..8d1e183d24 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -52,6 +52,12 @@ ${project.parent.version} test + + org.apache.hadoop + hadoop-auth + ${global_hadoop_version} + test + commons-validator commons-validator @@ -314,6 +320,12 @@ 2.0.2 test + + org.apache.metron + metron-test-utilities + ${project.parent.version} + test + commons-beanutils commons-beanutils From fe773389e50d47aae98ab09acbd676a768ed55db Mon Sep 17 00:00:00 2001 From: David Lyle Date: Wed, 8 Mar 2017 00:07:57 -0500 Subject: [PATCH 16/17] Attempt to capture any FATAL output by ambari-setup Don't run quick_dev role on ec2 buildout --- metron-deployment/amazon-ec2/run.sh | 2 +- metron-deployment/roles/ambari_master/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/amazon-ec2/run.sh b/metron-deployment/amazon-ec2/run.sh index 96c07af1d1..bdaabf0067 100755 --- a/metron-deployment/amazon-ec2/run.sh +++ b/metron-deployment/amazon-ec2/run.sh @@ -62,6 +62,6 @@ $DEPLOYDIR/../scripts/platform-info.sh >> $LOGFILE cd $DEPLOYDIR export EC2_INI_PATH=conf/ec2.ini ansible-playbook -i ec2.py playbook.yml \ - --skip-tags="sensor-stubs" \ + --skip-tags="quick_dev,sensor-stubs" \ --extra-vars="env=$ENV" \ $EXTRA_ARGS diff --git a/metron-deployment/roles/ambari_master/tasks/main.yml b/metron-deployment/roles/ambari_master/tasks/main.yml index 5b058bdf9a..28b89e81e7 100644 --- a/metron-deployment/roles/ambari_master/tasks/main.yml +++ b/metron-deployment/roles/ambari_master/tasks/main.yml @@ -36,7 +36,7 @@ - name: Setup ambari server shell: ambari-server setup -s && touch /etc/ambari-server/configured creates=/etc/ambari-server/configured register: ambari_server_setup - failed_when: ambari_server_setup.stderr + failed_when: "ambari_server_setup.stderr or 'FATAL' in ambari_server_setup.stdout" - name: Copy MPack to Ambari Host copy: From 6f8c9eecd7e86a839bc69fc63f513fd1e9c77eba Mon Sep 17 00:00:00 2001 From: David Lyle Date: Wed, 8 Mar 2017 10:11:22 -0500 Subject: [PATCH 17/17] Give hint that ES/Ambari will require quotes around boolean values. --- .../ELASTICSEARCH/2.3.3/configuration/elastic-site.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml index 36850907f1..cbcebdd502 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml @@ -29,7 +29,7 @@ masters_also_are_datanodes "false" - ES Masters and Slaves cannot be installed on the same nodes. Set this to true if you want the ES master nodes to serve as combined master/datanodes. + ES Masters and Slaves cannot be installed on the same nodes. Set this to "true" if you want the ES master nodes to serve as combined master/datanodes. Note: surround value in quotes. string