From 0805d20b95e3cdcf55bb0dfde91a08d8b9f58395 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Mon, 2 Oct 2017 12:51:23 -0600 Subject: [PATCH 01/59] Version changes for MPack --- .../ELASTICSEARCH/{2.3.3 => 5.6.2}/metainfo.xml | 6 +++--- .../{2.3.3 => 5.6.2}/repos/repoinfo.xml | 8 ++++---- .../KIBANA/{4.5.1 => 5.6.2}/metainfo.xml | 3 ++- .../{4.5.1 => 5.6.2}/quicklinks/quicklinks.json | 0 .../KIBANA/{4.5.1 => 5.6.2}/repos/repoinfo.xml | 16 ++++++++-------- .../configuration/elastic-env.xml | 0 .../configuration/elastic-site.xml | 0 .../configuration/elastic-sysconfig.xml | 0 .../ELASTICSEARCH/{2.3.3 => 5.6.2}/metainfo.xml | 4 ++-- .../{2.3.3 => 5.6.2}/package/scripts/elastic.py | 0 .../package/scripts/elastic_master.py | 0 .../package/scripts/elastic_slave.py | 0 .../{2.3.3 => 5.6.2}/package/scripts/params.py | 0 .../package/scripts/properties_config.py | 0 .../package/scripts/service_check.py | 0 .../{2.3.3 => 5.6.2}/package/scripts/slave.py | 0 .../package/scripts/status_params.py | 0 .../templates/elasticsearch.master.yaml.j2 | 0 .../templates/elasticsearch.slave.yaml.j2 | 0 .../{2.3.3 => 5.6.2}/quicklinks/quicklinks.json | 0 .../{2.3.3 => 5.6.2}/role_command_order.json | 0 .../configuration/kibana-env.xml | 0 .../configuration/kibana-site.xml | 0 .../KIBANA/{4.5.1 => 5.6.2}/metainfo.xml | 4 ++-- .../package/scripts/dashboard/__init__.py | 0 .../package/scripts/dashboard/dashboard.p | 0 .../package/scripts/dashboard/dashboardindex.py | 0 .../package/scripts/kibana_master.py | 0 .../{4.5.1 => 5.6.2}/package/scripts/params.py | 0 .../{4.5.1 => 5.6.2}/quicklinks/quicklinks.json | 0 .../metron-mpack/src/main/resources/mpack.json | 4 ++-- pom.xml | 1 + 32 files changed, 24 insertions(+), 22 deletions(-) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/metainfo.xml (89%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/repos/repoinfo.xml (80%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/{4.5.1 => 5.6.2}/metainfo.xml (91%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/{4.5.1 => 5.6.2}/quicklinks/quicklinks.json (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/{4.5.1 => 5.6.2}/repos/repoinfo.xml (73%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/configuration/elastic-env.xml (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/configuration/elastic-site.xml (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/configuration/elastic-sysconfig.xml (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/metainfo.xml (97%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/scripts/elastic.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/scripts/elastic_master.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/scripts/elastic_slave.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/scripts/params.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/scripts/properties_config.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/scripts/service_check.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/scripts/slave.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/scripts/status_params.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/templates/elasticsearch.master.yaml.j2 (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/package/templates/elasticsearch.slave.yaml.j2 (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/quicklinks/quicklinks.json (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/{2.3.3 => 5.6.2}/role_command_order.json (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/configuration/kibana-env.xml (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/configuration/kibana-site.xml (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/metainfo.xml (97%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/package/scripts/dashboard/__init__.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/package/scripts/dashboard/dashboard.p (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/package/scripts/dashboard/dashboardindex.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/package/scripts/kibana_master.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/package/scripts/params.py (100%) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/{4.5.1 => 5.6.2}/quicklinks/quicklinks.json (100%) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml similarity index 89% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml index e2e6cddb09..accf7da071 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml @@ -22,8 +22,8 @@ ELASTICSEARCH - 2.3.3 - common-services/ELASTICSEARCH/2.3.3 + 5.6.2 + common-services/ELASTICSEARCH/5.6.2 - \ No newline at end of file + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml similarity index 80% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml index 0a9caac3c1..2c645185b0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml @@ -16,15 +16,15 @@ - https://packages.elastic.co/elasticsearch/2.x/centos - elastic-2.x + https://artifacts.elastic.co/packages/5.x/yum + elasticsearch-5.x ELASTICSEARCH - https://packages.elastic.co/elasticsearch/2.x/centos - elasticsearch-2.x + https://artifacts.elastic.co/packages/5.x/yum + elasticsearch-5.x ELASTICSEARCH diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml similarity index 91% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml index 494f71b355..8a4fba2873 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml @@ -22,7 +22,8 @@ KIBANA - common-services/KIBANA/4.5.1 + 5.6.2 + common-services/KIBANA/5.6.2 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml similarity index 73% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml index b05c2cfa09..d0bd284afe 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml @@ -16,25 +16,25 @@ - http://packages.elastic.co/kibana/4.5/centos - kibana-4.x + https://artifacts.elastic.co/packages/5.x/yum + kibana-5.x KIBANA - http://packages.elastic.co/curator/4/centos/6 - ES-Curator-4.x + http://packages.elastic.co/curator/5/centos/6 + ES-Curator-5.x CURATOR - http://packages.elastic.co/kibana/4.5/centos - kibana-4.x + https://artifacts.elastic.co/packages/5.x/yum + kibana-5.x KIBANA - http://packages.elastic.co/curator/4/centos/7 - ES-Curator-4.x + http://packages.elastic.co/curator/5/centos/7 + ES-Curator-5.x CURATOR diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml similarity index 97% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml index 4373e381db..3783d31eab 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml @@ -22,7 +22,7 @@ ELASTICSEARCH Elasticsearch Indexing and Search - 2.3.3 + 5.6.2 ES_MASTER @@ -52,7 +52,7 @@ any - elasticsearch-2.3.3 + elasticsearch-5.6.2 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml similarity index 97% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml index f59109c8c7..034f71cfa8 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml @@ -22,7 +22,7 @@ KIBANA Kibana Kibana Dashboard - 4.5.1 + 5.6.2 KIBANA_MASTER @@ -54,7 +54,7 @@ python-elasticsearch - kibana-4.5.1 + kibana-5.6.2 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/__init__.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/__init__.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard.p similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard.p diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json similarity index 100% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json index 7c1b92d19b..e79d41fc37 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json @@ -25,7 +25,7 @@ "service_versions_map": [ { "service_name" : "KIBANA", - "service_version" : "4.5.1", + "service_version" : "5.6.2", "applicable_stacks" : [ { "stack_name" : "HDP", @@ -43,7 +43,7 @@ }, { "service_name" : "ELASTICSEARCH", - "service_version" : "2.3.3", + "service_version" : "5.6.2", "applicable_stacks" : [ { "stack_name" : "HDP", diff --git a/pom.xml b/pom.xml index 3f0af7e6a2..1e142e5e3d 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,7 @@ ${base_hadoop_version} ${base_hbase_version} ${base_flume_version} + 2.3.3 1.1.1 3.0.2 From 889c1bd5d24ee359d7d8113f500539ea10750a40 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 4 Oct 2017 08:55:08 -0600 Subject: [PATCH 02/59] save demo code --- .../elasticsearch-shaded562/pom.xml | 144 ++++ .../src/main/resources/META-INF/LICENSE | 441 ++++++++++++ .../src/main/resources/META-INF/NOTICE | 19 + .../metron-elasticsearch-test/pom.xml | 60 ++ .../src/main/assembly/assembly.xml | 54 ++ .../src/main/config/elasticsearch.properties | 47 ++ .../main/config/elasticsearch.properties.j2 | 49 ++ .../src/main/resources/META-INF/LICENSE | 669 ++++++++++++++++++ .../src/main/resources/META-INF/NOTICE | 99 +++ .../scripts/start_elasticsearch_topology.sh | 22 + .../metron/elasticsearch/IndexingTest.java | 110 +++ .../src/test/resources/log4j.properties | 24 + metron-platform/pom.xml | 2 + 13 files changed, 1740 insertions(+) create mode 100644 metron-platform/elasticsearch-shaded562/pom.xml create mode 100644 metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/LICENSE create mode 100644 metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/NOTICE create mode 100644 metron-platform/metron-elasticsearch-test/pom.xml create mode 100644 metron-platform/metron-elasticsearch-test/src/main/assembly/assembly.xml create mode 100644 metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties create mode 100644 metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties.j2 create mode 100644 metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/LICENSE create mode 100644 metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/NOTICE create mode 100755 metron-platform/metron-elasticsearch-test/src/main/scripts/start_elasticsearch_topology.sh create mode 100644 metron-platform/metron-elasticsearch-test/src/test/java/org/apache/metron/elasticsearch/IndexingTest.java create mode 100644 metron-platform/metron-elasticsearch-test/src/test/resources/log4j.properties diff --git a/metron-platform/elasticsearch-shaded562/pom.xml b/metron-platform/elasticsearch-shaded562/pom.xml new file mode 100644 index 0000000000..95f46aeecb --- /dev/null +++ b/metron-platform/elasticsearch-shaded562/pom.xml @@ -0,0 +1,144 @@ + + + + + metron-platform + org.apache.metron + 0.4.1 + + 4.0.0 + elasticsearch-shaded562 + elasticsearch-shaded562 + https://metron.apache.org/ + + + com.google.guava + guava + 18.0 + + + org.elasticsearch + elasticsearch + 5.6.2 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + + + com.fasterxml.jackson.core + jackson-core + + + + + com.fasterxml.jackson.core + jackson-core + ${global_jackson_version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + ${global_jackson_version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + ${global_jackson_version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + ${global_jackson_version} + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${global_shade_version} + + true + + + + package + + shade + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + com.google.common + org.apache.metron.guava.elasticsearch-shaded + + + + + storm:storm-core:* + storm:storm-lib:* + org.slf4j.impl* + org.slf4j:slf4j-log4j* + + + + + + .yaml + LICENSE.txt + ASL2.0 + NOTICE.txt + + + + + + + + + + + + + + + + diff --git a/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/LICENSE b/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/LICENSE new file mode 100644 index 0000000000..3bcbfafac9 --- /dev/null +++ b/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/LICENSE @@ -0,0 +1,441 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------ +------------------------------------------------------------------------------------ + Public Domain +------------------------------------------------------------------------------------ + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + + +------------------------------------------------------------------------------------ + BSD +------------------------------------------------------------------------------------ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See http://project.carrot2.org/license.html. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/). + + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------ + MIT +------------------------------------------------------------------------------------ + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------------ + Creative Commons +------------------------------------------------------------------------------------ +This product bundles jsr166e 1.1.0, which is available under a "Creative Commons License" license. For details, see http://github.com/twitter/jsr166e + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/NOTICE b/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000..c0209bba9a --- /dev/null +++ b/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/NOTICE @@ -0,0 +1,19 @@ + +elasticsearch-shaded +Copyright 2006-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + diff --git a/metron-platform/metron-elasticsearch-test/pom.xml b/metron-platform/metron-elasticsearch-test/pom.xml new file mode 100644 index 0000000000..923fc66e16 --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/pom.xml @@ -0,0 +1,60 @@ + + + + + 4.0.0 + + org.apache.metron + metron-platform + 0.4.1 + + metron-elasticsearch-test + metron-elasticsearch-test + https://metron.apache.org/ + + UTF-8 + UTF-8 + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 5.6.2 + + + org.apache.metron + metron-test-utilities + 0.4.1 + test + + + + commons-io + commons-io + 2.5 + + + diff --git a/metron-platform/metron-elasticsearch-test/src/main/assembly/assembly.xml b/metron-platform/metron-elasticsearch-test/src/main/assembly/assembly.xml new file mode 100644 index 0000000000..f8ce2da746 --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/src/main/assembly/assembly.xml @@ -0,0 +1,54 @@ + + + + archive + + tar.gz + + false + + + ${project.basedir}/src/main/config + config + true + + **/*.formatted + **/*.filtered + **/*.j2 + + 0644 + unix + + + ${project.basedir}/src/main/scripts + bin + true + + **/*.formatted + **/*.filtered + + 0755 + unix + true + + + ${project.basedir}/target + + ${project.artifactId}-${project.version}-uber.jar + + lib + true + + + diff --git a/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties new file mode 100644 index 0000000000..1620dfd4fb --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### Storm ##### +indexing.workers=1 +indexing.acker.executors=0 +topology.worker.childopts= +topology.auto-credentials=[''] +topology.max.spout.pending= + +##### Kafka ##### +kafka.zk=node1:2181 +kafka.broker=node1:6667 +kafka.security.protocol=PLAINTEXT + +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start=UNCOMMITTED_EARLIEST + +indexing.input.topic=indexing +indexing.error.topic=indexing + +##### Indexing ##### +indexing.writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter + +##### HDFS ##### +bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy +bolt.hdfs.rotation.policy.units=DAYS +bolt.hdfs.rotation.policy.count=1 +indexing.hdfs.output=/tmp/metron/enriched + +##### Parallelism ##### +kafka.spout.parallelism=1 +indexing.writer.parallelism=1 +hdfs.writer.parallelism=1 diff --git a/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties.j2 b/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties.j2 new file mode 100644 index 0000000000..acb0f59727 --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties.j2 @@ -0,0 +1,49 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +##### Storm ##### +indexing.workers={{indexing_workers}} +indexing.acker.executors={{indexing_acker_executors}} +topology.worker.childopts={{indexing_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{indexing_topology_max_spout_pending}} + +##### Kafka ##### +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +kafka.security.protocol={{kafka_security_protocol}} + +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{indexing_kafka_start}} + +indexing.input.topic={{indexing_input_topic}} +indexing.error.topic={{indexing_error_topic}} + +##### Indexing ##### +indexing.writer.class.name={{indexing_writer_class_name}} + +##### HDFS ##### +bolt.hdfs.rotation.policy={{bolt_hdfs_rotation_policy}} +bolt.hdfs.rotation.policy.units={{bolt_hdfs_rotation_policy_units}} +bolt.hdfs.rotation.policy.count={{bolt_hdfs_rotation_policy_count}} +indexing.hdfs.output={{metron_apps_indexed_hdfs_dir}} + +##### Parallelism ##### +kafka.spout.parallelism={{indexing_kafka_spout_parallelism}} +indexing.writer.parallelism={{indexing_writer_parallelism}} +hdfs.writer.parallelism={{hdfs_writer_parallelism}} diff --git a/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/LICENSE b/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/LICENSE new file mode 100644 index 0000000000..bead23a77d --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/LICENSE @@ -0,0 +1,669 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------ + +------------------------------------------------------------------------------------ + BSD +------------------------------------------------------------------------------------ + +This product bundles asm 3.1, which is available under a "BSD Software License" license. For details, see http://asm.ow2.org/ +This product bundles protobuf-java 2.5.0, which is available under a "BSD Software License" license. For details, see http://code.google.com/p/protobuf +This product bundles jsch 0.1.42, which is available under a "BSD Software License" license. For details, see http://www.jcraft.com/jsch/ +This product bundles paranamer 2.3, which is available under a "BSD Software License" license. For details, see https://github.com/paul-hammant/paranamer +This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni +This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ +This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net + +Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is +BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See http://project.carrot2.org/license.html. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/). + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------ + CDDL v1.1 +------------------------------------------------------------------------------------ + +This product bundles jersey-guice 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ +This product bundles jersey-client 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ +This product bundles jersey-core 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ +This product bundles jersey-json 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ +This product bundles jersey-server 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ +This product bundles jaxb-impl 2.2.3-1, which is available under a "Common Development and Distribution License" license. For details, see http://jaxb.java.net/ +This product bundles activation 1.1, which is available under a "Common Development and Distribution License v1.0" license. For details, see http://java.sun.com/products/javabeans/jaf/index.jsp +This product bundles jaxb-api 2.2.2, which is available under a "Common Development and Distribution License" license. For details, see https://jaxb.dev.java.net/ +This product bundles stax-api 1.0-2, which is available under a "Common Development and Distribution License v1.0" license. For details, see https://docs.oracle.com/javase/7/docs/api/javax/xml/stream/package-summary.html + +Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original +source code for this can be found at http://www.eclipse.org/jetty/downloads.php + +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 + 1. Definitions. + 1.1. "Contributor" means each individual or entity that + creates or contributes to the creation of Modifications. + 1.2. "Contributor Version" means the combination of the + Original Software, prior Modifications used by a + Contributor (if any), and the Modifications made by that + particular Contributor. + 1.3. "Covered Software" means (a) the Original Software, or + (b) Modifications, or (c) the combination of files + containing Original Software with files containing + Modifications, in each case including portions thereof. + 1.4. "Executable" means the Covered Software in any form + other than Source Code. + 1.5. "Initial Developer" means the individual or entity + that first makes Original Software available under this + License. + 1.6. "Larger Work" means a work which combines Covered + Software or portions thereof with code not governed by the + terms of this License. + 1.7. "License" means this document. + 1.8. "Licensable" means having the right to grant, to the + maximum extent possible, whether at the time of the initial + grant or subsequently acquired, any and all of the rights + conveyed herein. + 1.9. "Modifications" means the Source Code and Executable + form of any of the following: + A. Any file that results from an addition to, + deletion from or modification of the contents of a + file containing Original Software or previous + Modifications; + B. Any new file that contains any part of the + Original Software or previous Modification; or + C. Any new file that is contributed or otherwise made + available under the terms of this License. + 1.10. "Original Software" means the Source Code and + Executable form of computer software code that is + originally released under this License. + 1.11. "Patent Claims" means any patent claim(s), now owned + or hereafter acquired, including without limitation, + method, process, and apparatus claims, in any patent + Licensable by grantor. + 1.12. "Source Code" means (a) the common form of computer + software code in which modifications are made and (b) + associated documentation included in or with such code. + 1.13. "You" (or "Your") means an individual or a legal + entity exercising rights under, and complying with all of + the terms of, this License. For legal entities, "You" + includes any entity which controls, is controlled by, or is + under common control with You. For purposes of this + definition, "control" means (a) the power, direct or + indirect, to cause the direction or management of such + entity, whether by contract or otherwise, or (b) ownership + of more than fifty percent (50%) of the outstanding shares + or beneficial ownership of such entity. + 2. License Grants. + 2.1. The Initial Developer Grant. + Conditioned upon Your compliance with Section 3.1 below and + subject to third party intellectual property claims, the + Initial Developer hereby grants You a world-wide, + royalty-free, non-exclusive license: + (a) under intellectual property rights (other than + patent or trademark) Licensable by Initial Developer, + to use, reproduce, modify, display, perform, + sublicense and distribute the Original Software (or + portions thereof), with or without Modifications, + and/or as part of a Larger Work; and + (b) under Patent Claims infringed by the making, + using or selling of Original Software, to make, have + made, use, practice, sell, and offer for sale, and/or + otherwise dispose of the Original Software (or + portions thereof). + (c) The licenses granted in Sections 2.1(a) and (b) + are effective on the date Initial Developer first + distributes or otherwise makes the Original Software + available to a third party under the terms of this + License. + (d) Notwithstanding Section 2.1(b) above, no patent + license is granted: (1) for code that You delete from + the Original Software, or (2) for infringements + caused by: (i) the modification of the Original + Software, or (ii) the combination of the Original + Software with other software or devices. + 2.2. Contributor Grant. + Conditioned upon Your compliance with Section 3.1 below and + subject to third party intellectual property claims, each + Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + (a) under intellectual property rights (other than + patent or trademark) Licensable by Contributor to + use, reproduce, modify, display, perform, sublicense + and distribute the Modifications created by such + Contributor (or portions thereof), either on an + unmodified basis, with other Modifications, as + Covered Software and/or as part of a Larger Work; and + (b) under Patent Claims infringed by the making, + using, or selling of Modifications made by that + Contributor either alone and/or in combination with + its Contributor Version (or portions of such + combination), to make, use, sell, offer for sale, + have made, and/or otherwise dispose of: (1) + Modifications made by that Contributor (or portions + thereof); and (2) the combination of Modifications + made by that Contributor with its Contributor Version + (or portions of such combination). + (c) The licenses granted in Sections 2.2(a) and + 2.2(b) are effective on the date Contributor first + distributes or otherwise makes the Modifications + available to a third party. + (d) Notwithstanding Section 2.2(b) above, no patent + license is granted: (1) for any code that Contributor + has deleted from the Contributor Version; (2) for + infringements caused by: (i) third party + modifications of Contributor Version, or (ii) the + combination of Modifications made by that Contributor + with other software (except as part of the + Contributor Version) or other devices; or (3) under + Patent Claims infringed by Covered Software in the + absence of Modifications made by that Contributor. + 3. Distribution Obligations. + 3.1. Availability of Source Code. + Any Covered Software that You distribute or otherwise make + available in Executable form must also be made available in + Source Code form and that Source Code form must be + distributed only under the terms of this License. You must + include a copy of this License with every copy of the + Source Code form of the Covered Software You distribute or + otherwise make available. You must inform recipients of any + such Covered Software in Executable form as to how they can + obtain such Covered Software in Source Code form in a + reasonable manner on or through a medium customarily used + for software exchange. + 3.2. Modifications. + The Modifications that You create or to which You + contribute are governed by the terms of this License. You + represent that You believe Your Modifications are Your + original creation(s) and/or You have sufficient rights to + grant the rights conveyed by this License. + 3.3. Required Notices. + You must include a notice in each of Your Modifications + that identifies You as the Contributor of the Modification. + You may not remove or alter any copyright, patent or + trademark notices contained within the Covered Software, or + any notices of licensing or any descriptive text giving + attribution to any Contributor or the Initial Developer. + 3.4. Application of Additional Terms. + You may not offer or impose any terms on any Covered + Software in Source Code form that alters or restricts the + applicable version of this License or the recipients' + rights hereunder. You may choose to offer, and to charge a + fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Software. + However, you may do so only on Your own behalf, and not on + behalf of the Initial Developer or any Contributor. You + must make it absolutely clear that any such warranty, + support, indemnity or liability obligation is offered by + You alone, and You hereby agree to indemnify the Initial + Developer and every Contributor for any liability incurred + by the Initial Developer or such Contributor as a result of + warranty, support, indemnity or liability terms You offer. + 3.5. Distribution of Executable Versions. + You may distribute the Executable form of the Covered + Software under the terms of this License or under the terms + of a license of Your choice, which may contain terms + different from this License, provided that You are in + compliance with the terms of this License and that the + license for the Executable form does not attempt to limit + or alter the recipient's rights in the Source Code form + from the rights set forth in this License. If You + distribute the Covered Software in Executable form under a + different license, You must make it absolutely clear that + any terms which differ from this License are offered by You + alone, not by the Initial Developer or Contributor. You + hereby agree to indemnify the Initial Developer and every + Contributor for any liability incurred by the Initial + Developer or such Contributor as a result of any such terms + You offer. + 3.6. Larger Works. + You may create a Larger Work by combining Covered Software + with other code not governed by the terms of this License + and distribute the Larger Work as a single product. In such + a case, You must make sure the requirements of this License + are fulfilled for the Covered Software. + 4. Versions of the License. + 4.1. New Versions. + Sun Microsystems, Inc. is the initial license steward and + may publish revised and/or new versions of this License + from time to time. Each version will be given a + distinguishing version number. Except as provided in + Section 4.3, no one other than the license steward has the + right to modify this License. + 4.2. Effect of New Versions. + You may always continue to use, distribute or otherwise + make the Covered Software available under the terms of the + version of the License under which You originally received + the Covered Software. If the Initial Developer includes a + notice in the Original Software prohibiting it from being + distributed or otherwise made available under any + subsequent version of the License, You must distribute and + make the Covered Software available under the terms of the + version of the License under which You originally received + the Covered Software. Otherwise, You may also choose to + use, distribute or otherwise make the Covered Software + available under the terms of any subsequent version of the + License published by the license steward. + 4.3. Modified Versions. + When You are an Initial Developer and You want to create a + new license for Your Original Software, You may create and + use a modified version of this License if You: (a) rename + the license and remove any references to the name of the + license steward (except to note that the license differs + from this License); and (b) otherwise make it clear that + the license contains terms which differ from this License. + 5. DISCLAIMER OF WARRANTY. + COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" + BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, + INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED + SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR + PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND + PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY + COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE + INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF + ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF + WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF + ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS + DISCLAIMER. + 6. TERMINATION. + 6.1. This License and the rights granted hereunder will + terminate automatically if You fail to comply with terms + herein and fail to cure such breach within 30 days of + becoming aware of the breach. Provisions which, by their + nature, must remain in effect beyond the termination of + this License shall survive. + 6.2. If You assert a patent infringement claim (excluding + declaratory judgment actions) against Initial Developer or + a Contributor (the Initial Developer or Contributor against + whom You assert such claim is referred to as "Participant") + alleging that the Participant Software (meaning the + Contributor Version where the Participant is a Contributor + or the Original Software where the Participant is the + Initial Developer) directly or indirectly infringes any + patent, then any and all rights granted directly or + indirectly to You by such Participant, the Initial + Developer (if the Initial Developer is not the Participant) + and all Contributors under Sections 2.1 and/or 2.2 of this + License shall, upon 60 days notice from Participant + terminate prospectively and automatically at the expiration + of such 60 day notice period, unless if within such 60 day + period You withdraw Your claim with respect to the + Participant Software against such Participant either + unilaterally or pursuant to a written agreement with + Participant. + 6.3. In the event of termination under Sections 6.1 or 6.2 + above, all end user licenses that have been validly granted + by You or any distributor hereunder prior to termination + (excluding licenses granted to You by any distributor) + shall survive termination. + 7. LIMITATION OF LIABILITY. + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE + INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF + COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE + LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR + CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT + LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK + STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER + COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN + INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF + LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL + INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT + APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO + NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR + CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT + APPLY TO YOU. + 8. U.S. GOVERNMENT END USERS. + The Covered Software is a "commercial item," as that term is + defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial + computer software" (as that term is defined at 48 C.F.R. ¤ + 252.227-7014(a)(1)) and "commercial computer software + documentation" as such terms are used in 48 C.F.R. 12.212 (Sept. + 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 + through 227.7202-4 (June 1995), all U.S. Government End Users + acquire Covered Software with only those rights set forth herein. + This U.S. Government Rights clause is in lieu of, and supersedes, + any other FAR, DFAR, or other clause or provision that addresses + Government rights in computer software under this License. + 9. MISCELLANEOUS. + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the + extent necessary to make it enforceable. This License shall be + governed by the law of the jurisdiction specified in a notice + contained within the Original Software (except to the extent + applicable law, if any, provides otherwise), excluding such + jurisdiction's conflict-of-law provisions. Any litigation + relating to this License shall be subject to the jurisdiction of + the courts located in the jurisdiction and venue specified in a + notice contained within the Original Software, with the losing + party responsible for costs, including, without limitation, court + costs and reasonable attorneys' fees and expenses. The + application of the United Nations Convention on Contracts for the + International Sale of Goods is expressly excluded. Any law or + regulation which provides that the language of a contract shall + be construed against the drafter shall not apply to this License. + You agree that You alone are responsible for compliance with the + United States export administration regulations (and the export + control laws and regulation of any other countries) when You use, + distribute or otherwise make available any Covered Software. + 10. RESPONSIBILITY FOR CLAIMS. + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or + indirectly, out of its utilization of rights under this License + and You agree to work with Initial Developer and Contributors to + distribute such responsibility on an equitable basis. Nothing + herein is intended or shall be deemed to constitute any admission + of liability. + +------------------------------------------------------------------------------------ + MIT +------------------------------------------------------------------------------------ + +This product bundles jopt-simple 4.9, which is available under a "MIT Software License" license. For details, see http://jopt-simple.sourceforge.net +This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings +This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni +This product bundles slf4j-api 1.7.7, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org +This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org + +ICU4J, (under analysis/icu) is licensed under an MIT styles license + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------------ + Public Domain +------------------------------------------------------------------------------------ + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + + diff --git a/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/NOTICE b/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000..09d36cdcce --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/NOTICE @@ -0,0 +1,99 @@ + +metron-elasticsearch +Copyright 2006-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +ICU4J, (under analysis/icu) +Copyright (c) 1995-2008 International Business Machines Corporation and others + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + +The BracketFinder (package org.apache.commons.math3.optimization.univariate) +and PowellOptimizer (package org.apache.commons.math3.optimization.general) +classes are based on the Python code in module "optimize.py" (version 0.5) +developed by Travis E. Oliphant for the SciPy library (http://www.scipy.org/) +Copyright © 2003-2009 SciPy Developers. +=============================================================================== + +The LinearConstraint, LinearObjectiveFunction, LinearOptimizer, +RelationShip, SimplexSolver and SimplexTableau classes in package +org.apache.commons.math3.optimization.linear include software developed by +Benjamin McCann (http://www.benmccann.com) and distributed with +the following copyright: Copyright 2009 Google Inc. +=============================================================================== + +This product includes software developed by the +University of Chicago, as Operator of Argonne National +Laboratory. +The LevenbergMarquardtOptimizer class in package +org.apache.commons.math3.optimization.general includes software +translated from the lmder, lmpar and qrsolv Fortran routines +from the Minpack package +Minpack Copyright Notice (1999) University of Chicago. All rights reserved +=============================================================================== + +The GraggBulirschStoerIntegrator class in package +org.apache.commons.math3.ode.nonstiff includes software translated +from the odex Fortran routine developed by E. Hairer and G. Wanner. +Original source copyright: +Copyright (c) 2004, Ernst Hairer +=============================================================================== + +The EigenDecompositionImpl class in package +org.apache.commons.math3.linear includes software translated +from some LAPACK Fortran routines. Original source copyright: +Copyright (c) 1992-2008 The University of Tennessee. All rights reserved. +=============================================================================== + +The MersenneTwister class in package org.apache.commons.math3.random +includes software translated from the 2002-01-26 version of +the Mersenne-Twister generator written in C by Makoto Matsumoto and Takuji +Nishimura. Original source copyright: +Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, +All rights reserved +=============================================================================== + +The LocalizedFormatsTest class in the unit tests is an adapted version of +the OrekitMessagesTest class from the orekit library distributed under the +terms of the Apache 2 licence. Original source copyright: +Copyright 2010 CS Systèmes d'Information +=============================================================================== + +The HermiteInterpolator class and its corresponding test have been imported from +the orekit library distributed under the terms of the Apache 2 licence. Original +source copyright: +Copyright 2010-2012 CS Systèmes d'Information +=============================================================================== + +The creation of the package "o.a.c.m.analysis.integration.gauss" was inspired +by an original code donated by Sébastien Brisard. +=============================================================================== + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +This product includes software developed for Orekit by +CS Systèmes d'Information (http://www.c-s.fr/) +Copyright 2010-2012 CS Systèmes d'Information + +This project contains annotations derived from JCIP-ANNOTATIONS +Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net + +Objenesis +Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita + +Google Guice - Core Library +Copyright 2006-2011 Google, Inc. + +Google Guice - Extensions - Servlet +Copyright 2006-2011 Google, Inc. + diff --git a/metron-platform/metron-elasticsearch-test/src/main/scripts/start_elasticsearch_topology.sh b/metron-platform/metron-elasticsearch-test/src/main/scripts/start_elasticsearch_topology.sh new file mode 100755 index 0000000000..8ee7518d61 --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/src/main/scripts/start_elasticsearch_topology.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +METRON_VERSION=${project.version} +METRON_HOME=/usr/metron/$METRON_VERSION +TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar +storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties diff --git a/metron-platform/metron-elasticsearch-test/src/test/java/org/apache/metron/elasticsearch/IndexingTest.java b/metron-platform/metron-elasticsearch-test/src/test/java/org/apache/metron/elasticsearch/IndexingTest.java new file mode 100644 index 0000000000..01ea3bc4ec --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/src/test/java/org/apache/metron/elasticsearch/IndexingTest.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.elasticsearch; + +import java.io.File; +import java.io.IOException; +import org.apache.commons.io.FileUtils; +import org.apache.http.HttpHost; +import org.apache.metron.test.utils.UnitTestHelper; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeValidationException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class IndexingTest { + + private RestHighLevelClient client; + private static Node node; + private static final int httpPort = 9200; + + @BeforeClass + public static void oneTimeSetup() throws IOException { + File indexDir = UnitTestHelper.createTempDir(IndexingTest.class.toString()); + File logDir= new File(indexDir, "/logs"); + File dataDir= new File(indexDir, "/data"); + try { + cleanDir(logDir); + cleanDir(dataDir); + + } catch (IOException e) { + throw new RuntimeException("Unable to clean log or data directories", e); + } + + Settings.Builder settingsBuilder = Settings.builder() + .put("cluster.name", "metron") + .put("http.enabled", true) +// .put("http.port", httpPort) + .put("path.logs",logDir.getAbsolutePath()) + .put("path.data",dataDir.getAbsolutePath()) + .put("path.home", indexDir.getAbsoluteFile()) + .put("http.type", "http") + .put("transport.type", "local"); +// .put("index.number_of_shards", 1) +// .put("node.mode", "network") +// .put("index.number_of_replicas", 1); + node = new Node(settingsBuilder.build()); + wait(node, 60000); + } + + private static void cleanDir(File dir) throws IOException { + if(dir.exists()) { + FileUtils.deleteDirectory(dir); + } + dir.mkdirs(); + } + + private static void wait(Node node, long timeoutMillis) { + try { + node.start(); + ClusterHealthResponse chr = (ClusterHealthResponse) node.client() + .execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus( + ClusterHealthStatus.YELLOW).timeout(new TimeValue(timeoutMillis))).actionGet(); + if (chr != null && chr.isTimedOut()) { + throw new RuntimeException("cluster state is " + chr.getStatus().name() + + " and not " + ClusterHealthStatus.YELLOW.name() + + ", from here on, everything will fail!"); + } + } catch (NodeValidationException e) { + throw new RuntimeException("node validation exception"); + } + } + + @Before + public void setup() { +// RestClient restClient = RestClient.builder(new HttpHost("localhost", httpPort, "http")).build(); + RestClient restClient = RestClient.builder(new HttpHost("localhost", httpPort, "http")).build(); + client = new RestHighLevelClient(restClient); + } + + @Test + public void indexes_values() throws IOException { + System.out.println(client.info().getClusterName()); + } + +} diff --git a/metron-platform/metron-elasticsearch-test/src/test/resources/log4j.properties b/metron-platform/metron-elasticsearch-test/src/test/resources/log4j.properties new file mode 100644 index 0000000000..0d50388fff --- /dev/null +++ b/metron-platform/metron-elasticsearch-test/src/test/resources/log4j.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root logger option +log4j.rootLogger=ERROR, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml index 93ced81ca9..eeb379ef75 100644 --- a/metron-platform/pom.xml +++ b/metron-platform/pom.xml @@ -57,7 +57,9 @@ metron-writer metron-hbase elasticsearch-shaded + elasticsearch-shaded562 metron-elasticsearch + metron-elasticsearch-test metron-storm-kafka metron-storm-kafka-override From 42f82180208dd7235214810406596bfe896665ab Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 17 Oct 2017 06:12:43 -0600 Subject: [PATCH 03/59] Working through tests --- .../CURRENT/package/files/yaf_index.template | 87 +- .../SearchControllerIntegrationTest.java | 30 +- metron-platform/elasticsearch-shaded/pom.xml | 50 + .../elasticsearch-shaded562/pom.xml | 144 --- .../src/main/resources/META-INF/LICENSE | 441 --------- .../src/main/resources/META-INF/NOTICE | 19 - .../metron-data-management/pom.xml | 42 +- .../bulk/ElasticsearchDataPruner.java | 130 --- .../bulk/ElasticsearchDataPrunerRunner.java | 200 ---- .../ElasticsearchDataPrunerRunnerTest.java | 72 -- .../bulk/ElasticsearchDataPrunerTest.java | 855 ------------------ .../metron-elasticsearch-test/pom.xml | 60 -- .../src/main/assembly/assembly.xml | 54 -- .../src/main/config/elasticsearch.properties | 47 - .../main/config/elasticsearch.properties.j2 | 49 - .../src/main/resources/META-INF/LICENSE | 669 -------------- .../src/main/resources/META-INF/NOTICE | 99 -- .../scripts/start_elasticsearch_topology.sh | 22 - .../metron/elasticsearch/IndexingTest.java | 110 --- .../src/test/resources/log4j.properties | 24 - metron-platform/metron-elasticsearch/pom.xml | 16 + .../elasticsearch/dao/ElasticsearchDao.java | 61 +- .../dao/ElasticsearchMetaAlertDao.java | 15 +- .../utils/ElasticsearchUtils.java | 20 +- .../writer/ElasticsearchWriter.java | 8 +- .../dao/ElasticsearchDaoTest.java | 31 +- .../ElasticsearchSearchIntegrationTest.java | 37 +- .../components/ElasticSearchComponent.java | 345 +++---- .../matcher/SearchRequestMatcher.java | 15 +- .../metron/indexing/dao/search/FieldType.java | 6 +- .../indexing/dao/SearchIntegrationTest.java | 305 ++++--- metron-platform/pom.xml | 2 - pom.xml | 5 +- 33 files changed, 639 insertions(+), 3431 deletions(-) delete mode 100644 metron-platform/elasticsearch-shaded562/pom.xml delete mode 100644 metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/LICENSE delete mode 100644 metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/NOTICE delete mode 100644 metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java delete mode 100644 metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java delete mode 100644 metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunnerTest.java delete mode 100644 metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java delete mode 100644 metron-platform/metron-elasticsearch-test/pom.xml delete mode 100644 metron-platform/metron-elasticsearch-test/src/main/assembly/assembly.xml delete mode 100644 metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties delete mode 100644 metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties.j2 delete mode 100644 metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/LICENSE delete mode 100644 metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/NOTICE delete mode 100755 metron-platform/metron-elasticsearch-test/src/main/scripts/start_elasticsearch_topology.sh delete mode 100644 metron-platform/metron-elasticsearch-test/src/test/java/org/apache/metron/elasticsearch/IndexingTest.java delete mode 100644 metron-platform/metron-elasticsearch-test/src/test/resources/log4j.properties diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template index d84235d672..e9a2cd0227 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template @@ -2,9 +2,6 @@ "template": "yaf_index*", "mappings": { "yaf_doc": { - "_timestamp": { - "enabled": true - }, "dynamic_templates": [ { "geo_location_point": { @@ -20,8 +17,8 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -30,8 +27,8 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -40,8 +37,8 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -50,8 +47,8 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -60,8 +57,8 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -105,7 +102,7 @@ { "threat_triage_reason": { "mapping": { - "type": "string" + "type": "text" }, "match": "threat.triage.rules:*:reason", "match_mapping_type": "*" @@ -114,7 +111,7 @@ { "threat_triage_name": { "mapping": { - "type": "string" + "type": "text" }, "match": "threat.triage.rules:*:name", "match_mapping_type": "*" @@ -127,8 +124,8 @@ "format": "epoch_millis" }, "source:type": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "ip_dst_addr": { "type": "ip" @@ -157,55 +154,55 @@ "type": "double" }, "proto": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "sip": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "sp": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "dip": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "dp": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "iflags": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "uflags": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "riflags": { - "type": "string" + "type": "text" }, "ruflags": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "isn": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "risn": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "tag": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "rtag": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "pkt": { "type": "integer" @@ -220,11 +217,11 @@ "type": "integer" }, "app": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "end-reason": { - "type": "string" + "type": "text" }, "alert": { "type": "nested" diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java index ca7f209c8e..d7cf2b0d18 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java @@ -17,8 +17,19 @@ */ package org.apache.metron.rest.controller; +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.hamcrest.Matchers.hasSize; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import com.google.common.collect.ImmutableMap; -import org.apache.metron.hbase.mock.MockHBaseTableProvider; +import java.util.HashMap; +import java.util.Map; import org.apache.metron.indexing.dao.InMemoryDao; import org.apache.metron.indexing.dao.SearchIntegrationTest; import org.apache.metron.indexing.dao.search.FieldType; @@ -37,19 +48,6 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; -import static org.hamcrest.Matchers.hasSize; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; -import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(TEST_PROFILE) @@ -257,12 +255,12 @@ public void test() throws Exception { private void loadColumnTypes() throws ParseException { Map> columnTypes = new HashMap<>(); Map broTypes = new HashMap<>(); - broTypes.put("common_string_field", FieldType.STRING); + broTypes.put("common_string_field", FieldType.TEXT); broTypes.put("common_integer_field", FieldType.INTEGER); broTypes.put("bro_field", FieldType.BOOLEAN); broTypes.put("duplicate_field", FieldType.DATE); Map snortTypes = new HashMap<>(); - snortTypes.put("common_string_field", FieldType.STRING); + snortTypes.put("common_string_field", FieldType.TEXT); snortTypes.put("common_integer_field", FieldType.INTEGER); snortTypes.put("snort_field", FieldType.DOUBLE); snortTypes.put("duplicate_field", FieldType.LONG); diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index bbf96a08cb..0e3cbe420c 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -29,6 +29,52 @@ guava 18.0 + + + org.elasticsearch.client + transport + ${global_elasticsearch_version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + + + com.fasterxml.jackson.core + jackson-core + + + org.elasticsearch elasticsearch @@ -50,6 +96,10 @@ com.fasterxml.jackson.core jackson-core + + org.apache.logging.log4j + log4j-api + diff --git a/metron-platform/elasticsearch-shaded562/pom.xml b/metron-platform/elasticsearch-shaded562/pom.xml deleted file mode 100644 index 95f46aeecb..0000000000 --- a/metron-platform/elasticsearch-shaded562/pom.xml +++ /dev/null @@ -1,144 +0,0 @@ - - - - - metron-platform - org.apache.metron - 0.4.1 - - 4.0.0 - elasticsearch-shaded562 - elasticsearch-shaded562 - https://metron.apache.org/ - - - com.google.guava - guava - 18.0 - - - org.elasticsearch - elasticsearch - 5.6.2 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - - - com.fasterxml.jackson.core - jackson-core - - - - - com.fasterxml.jackson.core - jackson-core - ${global_jackson_version} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - ${global_jackson_version} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${global_jackson_version} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - ${global_jackson_version} - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${global_shade_version} - - true - - - - package - - shade - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - com.google.common - org.apache.metron.guava.elasticsearch-shaded - - - - - storm:storm-core:* - storm:storm-lib:* - org.slf4j.impl* - org.slf4j:slf4j-log4j* - - - - - - .yaml - LICENSE.txt - ASL2.0 - NOTICE.txt - - - - - - - - - - - - - - - - diff --git a/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/LICENSE b/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/LICENSE deleted file mode 100644 index 3bcbfafac9..0000000000 --- a/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/LICENSE +++ /dev/null @@ -1,441 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ------------------------------------------------------------------------------------- ------------------------------------------------------------------------------------- - Public Domain ------------------------------------------------------------------------------------- - -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - - ------------------------------------------------------------------------------------- - BSD ------------------------------------------------------------------------------------- - -This product contains a modified version of 'JZlib', a re-implementation of -zlib in pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD Style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product contains a modified version of 'Webbit', a Java event based -WebSocket and HTTP server: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - - -This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin -g Package (jaspell): http://jaspell.sourceforge.net/ -License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) - -The KStem stemmer in - analysis/common/src/org/apache/lucene/analysis/en -was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) -under the BSD-license. - -The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default -stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: -analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt -See http://members.unine.ch/jacques.savoy/clef/index.html. - -The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers -(common) are based on BSD-licensed reference implementations created by Jacques Savoy and -Ljiljana Dolamic. These files reside in: -analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java - -The Stempel analyzer (stempel) includes BSD-licensed software developed -by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, -and Edmond Nolan. - -The Polish analyzer (stempel) comes with a default -stopword list that is BSD-licensed created by the Carrot2 project. The file resides -in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. -See http://project.carrot2.org/license.html. - -The Morfologik analyzer (morfologik) includes BSD-licensed software -developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/). - - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ------------------------------------------------------------------------------------- - MIT ------------------------------------------------------------------------------------- - -The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were -automatically generated with the moman/finenight FSA library, created by -Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, -see http://sites.google.com/site/rrettesite/moman and -http://bitbucket.org/jpbarrette/moman/overview/ - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------------- - Creative Commons ------------------------------------------------------------------------------------- -This product bundles jsr166e 1.1.0, which is available under a "Creative Commons License" license. For details, see http://github.com/twitter/jsr166e - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. diff --git a/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/NOTICE b/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/NOTICE deleted file mode 100644 index c0209bba9a..0000000000 --- a/metron-platform/elasticsearch-shaded562/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,19 +0,0 @@ - -elasticsearch-shaded -Copyright 2006-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -The SmartChineseAnalyzer source code (smartcn) was -provided by Xiaoping Gao and copyright 2009 by www.imdict.net. - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2011 The Netty Project - diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml index 3fccc0afea..62de1dc536 100644 --- a/metron-platform/metron-data-management/pom.xml +++ b/metron-platform/metron-data-management/pom.xml @@ -231,11 +231,11 @@ httpclient ${httpcore.version} - + org.hamcrest hamcrest-all @@ -248,25 +248,25 @@ 2.1.14 test - - org.elasticsearch - elasticsearch - ${global_elasticsearch_version} - test-jar - test - - - org.apache.lucene - lucene-test-framework - ${lucene.test.version} - test - - - org.apache.lucene - lucene-core - ${lucene.test.version} - - + + org.apache.hadoop hadoop-hdfs ${global_hadoop_version} diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java deleted file mode 100644 index ce543456e6..0000000000 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.dataloads.bulk; - -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.Iterator; -import org.apache.commons.collections.IteratorUtils; -import org.apache.metron.common.configuration.Configuration; -import org.elasticsearch.client.AdminClient; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ElasticsearchDataPruner extends DataPruner { - - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String defaultDateFormat = "yyyy.MM.dd.HH"; - private String indexPattern; - private SimpleDateFormat dateFormat; - protected Client indexClient = null; - protected Configuration configuration; - - private Predicate filterWithRegex = new Predicate() { - - @Override - public boolean apply(String str) { - - try { - String dateString = str.substring(indexPattern.length()); - Date indexCreateDate = dateFormat.parse(dateString); - long indexCreatedDate = indexCreateDate.getTime(); - if (indexCreatedDate >= firstTimeMillis && indexCreatedDate < lastTimeMillis) { - return true; - } - } catch (ParseException e) { - LOG.error("Unable to parse date from {}", str.substring(indexPattern.length()), e); - } - - return false; - } - - }; - - public ElasticsearchDataPruner(Date startDate, Integer numDays,Configuration configuration, Client indexClient, String indexPattern) throws Exception { - - super(startDate, numDays, indexPattern); - - this.indexPattern = indexPattern; - this.dateFormat = new SimpleDateFormat(defaultDateFormat); - this.configuration = configuration; - this.indexClient = indexClient; - - - } - - @Override - public Long prune() throws IOException { - - try { - - configuration.update(); - - } - catch(Exception e) { - LOG.error("Unable to update configs",e); - } - - String dateString = configuration.getGlobalConfig().get("es.date.format").toString(); - - if( null != dateString ){ - dateFormat = new SimpleDateFormat(dateString); - } - - ImmutableOpenMap allIndices = indexClient.admin().cluster().prepareState().get().getState().getMetaData().getIndices(); - Iterable indicesForDeletion = getFilteredIndices(allIndices); - Object[] indexArray = IteratorUtils.toArray(indicesForDeletion.iterator()); - - if(indexArray.length > 0) { - String[] indexStringArray = new String[indexArray.length]; - System.arraycopy(indexArray, 0, indexStringArray, 0, indexArray.length); - deleteIndex(indexClient.admin(), indexStringArray); - } - - return (long) indexArray.length; - - } - - public boolean deleteIndex(AdminClient adminClient, String... index) { - - boolean isAcknowledged = adminClient.indices().delete(adminClient.indices().prepareDelete(index).request()).actionGet().isAcknowledged(); - return isAcknowledged; - - } - - protected Iterable getFilteredIndices(ImmutableOpenMap indices) { - - String[] returnedIndices = new String[indices.size()]; - Iterator it = indices.keysIt(); - System.arraycopy(IteratorUtils.toArray(it), 0, returnedIndices, 0, returnedIndices.length); - Iterable matches = Iterables.filter(Arrays.asList(returnedIndices), filterWithRegex); - - return matches; - - } - -} diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java deleted file mode 100644 index 5d2f0f1ad2..0000000000 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.dataloads.bulk; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.net.InetAddress; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; -import org.apache.curator.RetryPolicy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.metron.common.configuration.Configuration; -import org.apache.metron.common.utils.ErrorUtils; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ElasticsearchDataPrunerRunner { - - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static void main(String... argv) throws IOException, java.text.ParseException, ClassNotFoundException, InterruptedException { - - /** - * Example - * start=$(date -d '30 days ago' +%m/%d/%Y) - * yarn jar Metron-DataLoads-{VERSION}.jar org.apache.metron.dataloads.bulk.ElasticsearchDataPrunerRunner -i host1:9300 -p '/bro_index_' -s $(date -d '30 days ago' +%m/%d/%Y) -n 1; - * echo ${start} - **/ - - Options options = buildOptions(); - Options help = new Options(); - TransportClient client = null; - - Option o = new Option("h", "help", false, "This screen"); - o.setRequired(false); - help.addOption(o); - - - - try { - - CommandLine cmd = checkOptions(help,options, argv); - - String start = cmd.getOptionValue("s"); - Date startDate = new SimpleDateFormat("MM/dd/yyyy").parse(start); - - Integer numDays = Integer.parseInt(cmd.getOptionValue("n")); - String indexPrefix = cmd.getOptionValue("p"); - - LOG.debug("Running prune with args: {} {}", startDate, numDays); - - Configuration configuration = null; - - if( cmd.hasOption("z")){ - - RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); - CuratorFramework framework = CuratorFrameworkFactory.newClient(cmd.getOptionValue("z"),retryPolicy); - framework.start(); - configuration = new Configuration(framework); - - } else if ( cmd.hasOption("c") ){ - - String resourceFile = cmd.getOptionValue("c"); - configuration = new Configuration(Paths.get(resourceFile)); - - } else { - ErrorUtils.RuntimeErrors.ILLEGAL_ARG.throwRuntime("Unable to finish setting up configuration - z or c option is required."); - } - - configuration.update(); - - Map globalConfiguration = configuration.getGlobalConfig(); - - Settings.Builder settingsBuilder = Settings.settingsBuilder(); - settingsBuilder.put("cluster.name", globalConfiguration.get("es.clustername")); - settingsBuilder.put("curatorFramework.transport.ping_timeout","500s"); - Settings settings = settingsBuilder.build(); - client = TransportClient.builder().settings(settings).build() - .addTransportAddress( - new InetSocketTransportAddress(InetAddress.getByName(globalConfiguration.get("es.ip").toString()), Integer.parseInt(globalConfiguration.get("es.port").toString()) ) - ); - - DataPruner pruner = new ElasticsearchDataPruner(startDate, numDays, configuration, client, indexPrefix); - - LOG.info("Pruned {} indices from {}:{}/{}", pruner.prune(), globalConfiguration.get("es.ip"), globalConfiguration.get("es.port"), indexPrefix); - } catch (Exception e) { - - e.printStackTrace(); - System.exit(-1); - - } finally { - - if( null != client) { - client.close(); - } - - } - - } - - public static CommandLine checkOptions(Options help, Options options, String ... argv) throws ParseException { - - CommandLine cmd = null; - CommandLineParser parser = new PosixParser(); - - - try { - - cmd = parser.parse(help,argv,true); - - if( cmd.getOptions().length > 0){ - final HelpFormatter usageFormatter = new HelpFormatter(); - usageFormatter.printHelp("ElasticsearchDataPrunerRunner", null, options, null, true); - System.exit(0); - } - - cmd = parser.parse(options, argv); - - } catch (ParseException e) { - - final HelpFormatter usageFormatter = new HelpFormatter(); - usageFormatter.printHelp("ElasticsearchDataPrunerRunner", null, options, null, true); - throw e; - - } - - - if( (cmd.hasOption("z") && cmd.hasOption("c")) || (!cmd.hasOption("z") && !cmd.hasOption("c")) ){ - - System.err.println("One (only) of zookeeper-hosts or config-location is required"); - final HelpFormatter usageFormatter = new HelpFormatter(); - usageFormatter.printHelp("ElasticsearchDataPrunerRunner", null, options, null, true); - throw new RuntimeException("Must specify zookeeper-hosts or config-location, but not both"); - - } - - return cmd; - } - - public static Options buildOptions(){ - - Options options = new Options(); - - Option o = new Option("s", "start-date", true, "Starting Date (MM/DD/YYYY)"); - o.setArgName("START_DATE"); - o.setRequired(true); - options.addOption(o); - - o = new Option("n", "numdays", true, "Number of days back to purge"); - o.setArgName("NUMDAYS"); - o.setRequired(true); - options.addOption(o); - - o = new Option("p", "index-prefix", true, "Index prefix - e.g. bro_index_"); - o.setArgName("PREFIX"); - o.setRequired(true); - options.addOption(o); - - o = new Option("c", "config-location", true, "Directory Path - e.g. /path/to/config/dir"); - o.setArgName("CONFIG"); - o.setRequired(false); - options.addOption(o); - - o = new Option("z", "zookeeper-hosts", true, "Zookeeper URL - e.g. zkhost1:2181,zkhost2:2181,zkhost3:2181"); - o.setArgName("PREFIX"); - o.setRequired(false); - options.addOption(o); - - return options; - } -} diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunnerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunnerTest.java deleted file mode 100644 index 5f32bee53c..0000000000 --- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunnerTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.dataloads.bulk; - -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.FileDescriptor; -import java.io.FileOutputStream; -import java.io.PrintStream; - -public class ElasticsearchDataPrunerRunnerTest { - - private Options options; - private Options help; - - private ByteArrayOutputStream outContent; - private ByteArrayOutputStream errContent; - - @Before - public void setUp(){ - - options = ElasticsearchDataPrunerRunner.buildOptions(); - help = new Options(); - - Option o = new Option("h", "help", false, "This screen"); - o.setRequired(false); - help.addOption(o); - - outContent = new ByteArrayOutputStream(); - errContent = new ByteArrayOutputStream(); - - System.setOut(new PrintStream(outContent)); - System.setErr(new PrintStream(errContent)); - - } - - @Test(expected = RuntimeException.class) - public void testThrowsWithoutZookeeperOrConfigLocation() throws Exception { - - String[] args = new String[]{"-n","30","-p","sensor_index","-s","03/30/2016"}; - ElasticsearchDataPrunerRunner.checkOptions(help,options,args); - - } - - @Test(expected = RuntimeException.class) - public void testThrowsWithZookeeperAndConfiguration() throws Exception { - - String[] args = new String[]{"-n","30","-p","sensor_index","-s","03/30/2016"}; - ElasticsearchDataPrunerRunner.checkOptions(help,options,args); - - } - -} diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java deleted file mode 100644 index df485f069d..0000000000 --- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java +++ /dev/null @@ -1,855 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.dataloads.bulk; - -import com.carrotsearch.hppc.ObjectObjectHashMap; -import org.apache.commons.collections.IteratorUtils; -import org.apache.metron.TestConstants; -import org.apache.metron.common.configuration.Configuration; -import org.easymock.EasyMock; -import org.elasticsearch.action.*; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder; -import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse; -import org.elasticsearch.action.admin.indices.flush.*; -import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; -import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; -import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.admin.indices.mapping.get.*; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; -import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; -import org.elasticsearch.action.admin.indices.recovery.RecoveryRequestBuilder; -import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse; -import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBuilder; -import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest; -import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequestBuilder; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; -import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusRequest; -import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusRequestBuilder; -import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse; -import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest; -import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequestBuilder; -import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeResponse; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; -import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequest; -import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequestBuilder; -import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerResponse; -import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersRequest; -import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersRequestBuilder; -import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersResponse; -import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequest; -import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequestBuilder; -import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerResponse; -import org.elasticsearch.client.AdminClient; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.threadpool.ThreadPool; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Date; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.powermock.api.easymock.PowerMock.replayAll; -import static org.powermock.api.easymock.PowerMock.verifyAll; - -@RunWith(PowerMockRunner.class) -@PrepareForTest(DeleteIndexResponse.class) -public class ElasticsearchDataPrunerTest { - - private Date testDate; - private DateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd.HH"); - private Configuration configuration; - - private Client indexClient = mock(Client.class); - private AdminClient adminClient = mock(AdminClient.class); - private IndicesAdminClient indicesAdminClient = new TestIndicesAdminClient(); - private DeleteIndexRequestBuilder deleteIndexRequestBuilder = mock(DeleteIndexRequestBuilder.class); - private DeleteIndexRequest deleteIndexRequest = mock(DeleteIndexRequest.class); - private ActionFuture deleteIndexAction = mock(ActionFuture.class); - private DeleteIndexResponse deleteIndexResponse = PowerMock.createMock(DeleteIndexResponse.class); - - - private ByteArrayOutputStream outContent; - private ByteArrayOutputStream errContent; - - @Before - public void setUp() throws Exception { - - Calendar calendar = Calendar.getInstance(); - calendar.set(Calendar.MONTH, Calendar.MARCH); - calendar.set(Calendar.YEAR, 2016); - calendar.set(Calendar.DATE, 31); - calendar.set(Calendar.HOUR_OF_DAY, 0); - calendar.set(Calendar.MINUTE, 0); - calendar.set(Calendar.SECOND, 0); - calendar.set(Calendar.MILLISECOND,0); - testDate = calendar.getTime(); - - when(indexClient.admin()).thenReturn(adminClient); - when(adminClient.indices()).thenReturn(indicesAdminClient); - when(deleteIndexRequestBuilder.request()).thenReturn(deleteIndexRequest); - when(deleteIndexAction.actionGet()).thenReturn(deleteIndexResponse); - - File resourceFile = new File(TestConstants.SAMPLE_CONFIG_PATH); - Path resourcePath = Paths.get(resourceFile.getCanonicalPath()); - - configuration = new Configuration(resourcePath); - - outContent = new ByteArrayOutputStream(); - errContent = new ByteArrayOutputStream(); - - System.setOut(new PrintStream(outContent)); - System.setErr(new PrintStream(errContent)); - - } - - @Test(expected = IndexNotFoundException.class) - public void testWillThrowOnMissingIndex() throws Exception { - - ((TestIndicesAdminClient)indicesAdminClient).throwMissingIndex = true; - ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(testDate, 30, configuration, indexClient,"*"); - pruner.deleteIndex(adminClient, "baz"); - ((TestIndicesAdminClient)indicesAdminClient).throwMissingIndex = false; - - } - - @Test - public void testDeletesCorrectIndexes() throws Exception { - - //Mock Cluster Admin - ClusterAdminClient clusterAdminClient = mock(ClusterAdminClient.class); - ClusterStateRequestBuilder clusterStateRequestBuilder = mock(ClusterStateRequestBuilder.class); - ClusterStateResponse clusterStateResponse = mock(ClusterStateResponse.class); - ClusterState clusterState = mock(ClusterState.class); - ObjectObjectHashMap clusterIndexes = new ObjectObjectHashMap(); - MetaData clusterMetadata = mock(MetaData.class); - when(adminClient.cluster()).thenReturn(clusterAdminClient); - when(clusterAdminClient.prepareState()).thenReturn(clusterStateRequestBuilder); - when(clusterStateRequestBuilder.get()).thenReturn(clusterStateResponse); - when(clusterStateResponse.getState()).thenReturn(clusterState); - when(clusterState.getMetaData()).thenReturn(clusterMetadata); - - int numDays = 5; - - Date indexDate = new Date(); - - indexDate.setTime(testDate.getTime() - TimeUnit.DAYS.toMillis(numDays)); - - for (int i = 0; i < numDays * 24; i++) { - - String indexName = "sensor_index_" + dateFormat.format(indexDate); - clusterIndexes.put(indexName, null); - indexDate.setTime(indexDate.getTime() + TimeUnit.HOURS.toMillis(1)); - - } - - when(clusterMetadata.getIndices()).thenReturn(ImmutableOpenMap.copyOf(clusterIndexes)); - - - EasyMock.expect(deleteIndexResponse.isAcknowledged()).andReturn(true); - - replayAll(); - ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(testDate, 1, configuration, indexClient, "sensor_index_"); - pruner.indexClient = indexClient; - Long deleteCount = pruner.prune(); - assertEquals("Should have pruned 24 indices", 24L, deleteCount.longValue()); - verifyAll(); - - } - - @Test - public void testFilter() throws Exception { - - ObjectObjectHashMap indexNames = new ObjectObjectHashMap(); - SimpleDateFormat dateChecker = new SimpleDateFormat("yyyyMMdd"); - int numDays = 5; - String[] expectedIndices = new String[24]; - Date indexDate = new Date(); - - indexDate.setTime(testDate.getTime() - TimeUnit.DAYS.toMillis(numDays)); - - for (int i = 0, j=0; i < numDays * 24; i++) { - - String indexName = "sensor_index_" + dateFormat.format(indexDate); - //Delete 20160330 - if( dateChecker.format(indexDate).equals("20160330") ){ - expectedIndices[j++] = indexName; - } - - indexNames.put(indexName, null); - indexDate.setTime(indexDate.getTime() + TimeUnit.HOURS.toMillis(1)); - - } - - ImmutableOpenMap testIndices = ImmutableOpenMap.copyOf(indexNames); - - ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(testDate, 1, configuration, indexClient, "sensor_index_"); - pruner.indexClient = indexClient; - - Iterable filteredIndices = pruner.getFilteredIndices(testIndices); - - Object[] indexArray = IteratorUtils.toArray(filteredIndices.iterator()); - Arrays.sort(indexArray); - Arrays.sort(expectedIndices); - - assertArrayEquals(expectedIndices,indexArray); - - } - - class TestIndicesAdminClient implements IndicesAdminClient { - - public boolean throwMissingIndex = false; - - @Override - public ActionFuture delete(DeleteIndexRequest request) { - - if(throwMissingIndex){ - - throw new IndexNotFoundException("TEST EXCEPTION!"); - - } - - return deleteIndexAction; - - } - - - @Override - public ActionFuture exists(IndicesExistsRequest request) { - return null; - } - - @Override - public void exists(IndicesExistsRequest request, ActionListener listener) { - - } - - @Override - public IndicesExistsRequestBuilder prepareExists(String... indices) { - return null; - } - - @Override - public ActionFuture typesExists(TypesExistsRequest request) { - return null; - } - - @Override - public void typesExists(TypesExistsRequest request, ActionListener listener) { - - } - - @Override - public TypesExistsRequestBuilder prepareTypesExists(String... index) { - return null; - } - - @Override - public ActionFuture stats(IndicesStatsRequest request) { - return null; - } - - @Override - public void stats(IndicesStatsRequest request, ActionListener listener) { - - } - - @Override - public IndicesStatsRequestBuilder prepareStats(String... indices) { - return null; - } - - @Override - public ActionFuture recoveries(RecoveryRequest request) { - return null; - } - - @Override - public void recoveries(RecoveryRequest request, ActionListener listener) { - - } - - @Override - public RecoveryRequestBuilder prepareRecoveries(String... indices) { - return null; - } - - @Override - public ActionFuture segments(IndicesSegmentsRequest request) { - return null; - } - - @Override - public void segments(IndicesSegmentsRequest request, ActionListener listener) { - - } - - @Override - public IndicesSegmentsRequestBuilder prepareSegments(String... indices) { - return null; - } - - @Override - public ActionFuture shardStores(IndicesShardStoresRequest request) { - return null; - } - - @Override - public void shardStores(IndicesShardStoresRequest request, ActionListener listener) { - - } - - @Override - public IndicesShardStoreRequestBuilder prepareShardStores(String... indices) { - return null; - } - - @Override - public ActionFuture create(CreateIndexRequest request) { - return null; - } - - @Override - public void create(CreateIndexRequest request, ActionListener listener) { - - } - - @Override - public CreateIndexRequestBuilder prepareCreate(String index) { - return null; - } - - - @Override - public void delete(DeleteIndexRequest request, ActionListener listener) { - - } - - @Override - public DeleteIndexRequestBuilder prepareDelete(String... indices) { - return deleteIndexRequestBuilder; - } - - @Override - public ActionFuture close(CloseIndexRequest request) { - return null; - } - - @Override - public void close(CloseIndexRequest request, ActionListener listener) { - - } - - @Override - public CloseIndexRequestBuilder prepareClose(String... indices) { - return null; - } - - @Override - public ActionFuture open(OpenIndexRequest request) { - return null; - } - - @Override - public void open(OpenIndexRequest request, ActionListener listener) { - - } - - @Override - public OpenIndexRequestBuilder prepareOpen(String... indices) { - return null; - } - - @Override - public ActionFuture refresh(RefreshRequest request) { - return null; - } - - @Override - public void refresh(RefreshRequest request, ActionListener listener) { - - } - - @Override - public RefreshRequestBuilder prepareRefresh(String... indices) { - return null; - } - - @Override - public ActionFuture flush(FlushRequest request) { - return null; - } - - @Override - public void flush(FlushRequest request, ActionListener listener) { - - } - - @Override - public FlushRequestBuilder prepareFlush(String... indices) { - return null; - } - - @Override - public ActionFuture syncedFlush(SyncedFlushRequest request) { - return null; - } - - @Override - public void syncedFlush(SyncedFlushRequest request, ActionListener listener) { - - } - - @Override - public SyncedFlushRequestBuilder prepareSyncedFlush(String... indices) { - return null; - } - - @Override - public ActionFuture forceMerge(ForceMergeRequest request) { - return null; - } - - @Override - public void forceMerge(ForceMergeRequest request, ActionListener listener) { - - } - - @Override - public ForceMergeRequestBuilder prepareForceMerge(String... indices) { - return null; - } - - @Override - public ActionFuture upgrade(UpgradeRequest request) { - return null; - } - - @Override - public void upgrade(UpgradeRequest request, ActionListener listener) { - - } - - @Override - public UpgradeStatusRequestBuilder prepareUpgradeStatus(String... indices) { - return null; - } - - @Override - public ActionFuture upgradeStatus(UpgradeStatusRequest request) { - return null; - } - - @Override - public void upgradeStatus(UpgradeStatusRequest request, ActionListener listener) { - - } - - @Override - public UpgradeRequestBuilder prepareUpgrade(String... indices) { - return null; - } - - @Override - public void getMappings(GetMappingsRequest request, ActionListener listener) { - - } - - @Override - public ActionFuture getMappings(GetMappingsRequest request) { - return null; - } - - @Override - public GetMappingsRequestBuilder prepareGetMappings(String... indices) { - return null; - } - - @Override - public void getFieldMappings(GetFieldMappingsRequest request, ActionListener listener) { - - } - - @Override - public GetFieldMappingsRequestBuilder prepareGetFieldMappings(String... indices) { - return null; - } - - @Override - public ActionFuture getFieldMappings(GetFieldMappingsRequest request) { - return null; - } - - @Override - public ActionFuture putMapping(PutMappingRequest request) { - return null; - } - - @Override - public void putMapping(PutMappingRequest request, ActionListener listener) { - - } - - @Override - public PutMappingRequestBuilder preparePutMapping(String... indices) { - return null; - } - - @Override - public ActionFuture aliases(IndicesAliasesRequest request) { - return null; - } - - @Override - public void aliases(IndicesAliasesRequest request, ActionListener listener) { - - } - - @Override - public IndicesAliasesRequestBuilder prepareAliases() { - return null; - } - - @Override - public ActionFuture getAliases(GetAliasesRequest request) { - return null; - } - - @Override - public void getAliases(GetAliasesRequest request, ActionListener listener) { - - } - - @Override - public GetAliasesRequestBuilder prepareGetAliases(String... aliases) { - return null; - } - - @Override - public AliasesExistRequestBuilder prepareAliasesExist(String... aliases) { - return null; - } - - @Override - public ActionFuture aliasesExist(GetAliasesRequest request) { - return null; - } - - @Override - public void aliasesExist(GetAliasesRequest request, ActionListener listener) { - - } - - @Override - public ActionFuture getIndex(GetIndexRequest request) { - return null; - } - - @Override - public void getIndex(GetIndexRequest request, ActionListener listener) { - - } - - @Override - public GetIndexRequestBuilder prepareGetIndex() { - return null; - } - - @Override - public ActionFuture clearCache(ClearIndicesCacheRequest request) { - return null; - } - - @Override - public void clearCache(ClearIndicesCacheRequest request, ActionListener listener) { - - } - - @Override - public ClearIndicesCacheRequestBuilder prepareClearCache(String... indices) { - return null; - } - - @Override - public ActionFuture updateSettings(UpdateSettingsRequest request) { - return null; - } - - @Override - public void updateSettings(UpdateSettingsRequest request, ActionListener listener) { - - } - - @Override - public UpdateSettingsRequestBuilder prepareUpdateSettings(String... indices) { - return null; - } - - @Override - public ActionFuture analyze(AnalyzeRequest request) { - return null; - } - - @Override - public void analyze(AnalyzeRequest request, ActionListener listener) { - - } - - @Override - public AnalyzeRequestBuilder prepareAnalyze(@Nullable String index, String text) { - return null; - } - - @Override - public AnalyzeRequestBuilder prepareAnalyze(String text) { - return null; - } - - @Override - public AnalyzeRequestBuilder prepareAnalyze() { - return null; - } - - @Override - public ActionFuture putTemplate(PutIndexTemplateRequest request) { - return null; - } - - @Override - public void putTemplate(PutIndexTemplateRequest request, ActionListener listener) { - - } - - @Override - public PutIndexTemplateRequestBuilder preparePutTemplate(String name) { - return null; - } - - @Override - public ActionFuture deleteTemplate(DeleteIndexTemplateRequest request) { - return null; - } - - @Override - public void deleteTemplate(DeleteIndexTemplateRequest request, ActionListener listener) { - - } - - @Override - public DeleteIndexTemplateRequestBuilder prepareDeleteTemplate(String name) { - return null; - } - - @Override - public ActionFuture getTemplates(GetIndexTemplatesRequest request) { - return null; - } - - @Override - public void getTemplates(GetIndexTemplatesRequest request, ActionListener listener) { - - } - - @Override - public GetIndexTemplatesRequestBuilder prepareGetTemplates(String... name) { - return null; - } - - @Override - public ActionFuture validateQuery(ValidateQueryRequest request) { - return null; - } - - @Override - public void validateQuery(ValidateQueryRequest request, ActionListener listener) { - - } - - @Override - public ValidateQueryRequestBuilder prepareValidateQuery(String... indices) { - return null; - } - - @Override - public ActionFuture putWarmer(PutWarmerRequest request) { - return null; - } - - @Override - public void putWarmer(PutWarmerRequest request, ActionListener listener) { - - } - - @Override - public PutWarmerRequestBuilder preparePutWarmer(String name) { - return null; - } - - @Override - public ActionFuture deleteWarmer(DeleteWarmerRequest request) { - return null; - } - - @Override - public void deleteWarmer(DeleteWarmerRequest request, ActionListener listener) { - - } - - @Override - public DeleteWarmerRequestBuilder prepareDeleteWarmer() { - return null; - } - - @Override - public void getWarmers(GetWarmersRequest request, ActionListener listener) { - - } - - @Override - public ActionFuture getWarmers(GetWarmersRequest request) { - return null; - } - - @Override - public GetWarmersRequestBuilder prepareGetWarmers(String... indices) { - return null; - } - - @Override - public void getSettings(GetSettingsRequest request, ActionListener listener) { - - } - - @Override - public ActionFuture getSettings(GetSettingsRequest request) { - return null; - } - - @Override - public GetSettingsRequestBuilder prepareGetSettings(String... indices) { - return null; - } - - @Override - public > ActionFuture execute(Action action, Request request) { - return null; - } - - @Override - public > void execute(Action action, Request request, ActionListener listener) { - - } - - @Override - public > RequestBuilder prepareExecute(Action action) { - return null; - } - - @Override - public ThreadPool threadPool() { - return null; - } - } - -} diff --git a/metron-platform/metron-elasticsearch-test/pom.xml b/metron-platform/metron-elasticsearch-test/pom.xml deleted file mode 100644 index 923fc66e16..0000000000 --- a/metron-platform/metron-elasticsearch-test/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - 4.0.0 - - org.apache.metron - metron-platform - 0.4.1 - - metron-elasticsearch-test - metron-elasticsearch-test - https://metron.apache.org/ - - UTF-8 - UTF-8 - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - 5.6.2 - - - org.apache.metron - metron-test-utilities - 0.4.1 - test - - - - commons-io - commons-io - 2.5 - - - diff --git a/metron-platform/metron-elasticsearch-test/src/main/assembly/assembly.xml b/metron-platform/metron-elasticsearch-test/src/main/assembly/assembly.xml deleted file mode 100644 index f8ce2da746..0000000000 --- a/metron-platform/metron-elasticsearch-test/src/main/assembly/assembly.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - archive - - tar.gz - - false - - - ${project.basedir}/src/main/config - config - true - - **/*.formatted - **/*.filtered - **/*.j2 - - 0644 - unix - - - ${project.basedir}/src/main/scripts - bin - true - - **/*.formatted - **/*.filtered - - 0755 - unix - true - - - ${project.basedir}/target - - ${project.artifactId}-${project.version}-uber.jar - - lib - true - - - diff --git a/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties deleted file mode 100644 index 1620dfd4fb..0000000000 --- a/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -##### Storm ##### -indexing.workers=1 -indexing.acker.executors=0 -topology.worker.childopts= -topology.auto-credentials=[''] -topology.max.spout.pending= - -##### Kafka ##### -kafka.zk=node1:2181 -kafka.broker=node1:6667 -kafka.security.protocol=PLAINTEXT - -# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST -kafka.start=UNCOMMITTED_EARLIEST - -indexing.input.topic=indexing -indexing.error.topic=indexing - -##### Indexing ##### -indexing.writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter - -##### HDFS ##### -bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy -bolt.hdfs.rotation.policy.units=DAYS -bolt.hdfs.rotation.policy.count=1 -indexing.hdfs.output=/tmp/metron/enriched - -##### Parallelism ##### -kafka.spout.parallelism=1 -indexing.writer.parallelism=1 -hdfs.writer.parallelism=1 diff --git a/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties.j2 b/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties.j2 deleted file mode 100644 index acb0f59727..0000000000 --- a/metron-platform/metron-elasticsearch-test/src/main/config/elasticsearch.properties.j2 +++ /dev/null @@ -1,49 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -##### Storm ##### -indexing.workers={{indexing_workers}} -indexing.acker.executors={{indexing_acker_executors}} -topology.worker.childopts={{indexing_topology_worker_childopts}} -topology.auto-credentials={{topology_auto_credentials}} -topology.max.spout.pending={{indexing_topology_max_spout_pending}} - -##### Kafka ##### -kafka.zk={{zookeeper_quorum}} -kafka.broker={{kafka_brokers}} -kafka.security.protocol={{kafka_security_protocol}} - -# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST -kafka.start={{indexing_kafka_start}} - -indexing.input.topic={{indexing_input_topic}} -indexing.error.topic={{indexing_error_topic}} - -##### Indexing ##### -indexing.writer.class.name={{indexing_writer_class_name}} - -##### HDFS ##### -bolt.hdfs.rotation.policy={{bolt_hdfs_rotation_policy}} -bolt.hdfs.rotation.policy.units={{bolt_hdfs_rotation_policy_units}} -bolt.hdfs.rotation.policy.count={{bolt_hdfs_rotation_policy_count}} -indexing.hdfs.output={{metron_apps_indexed_hdfs_dir}} - -##### Parallelism ##### -kafka.spout.parallelism={{indexing_kafka_spout_parallelism}} -indexing.writer.parallelism={{indexing_writer_parallelism}} -hdfs.writer.parallelism={{hdfs_writer_parallelism}} diff --git a/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/LICENSE b/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/LICENSE deleted file mode 100644 index bead23a77d..0000000000 --- a/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/LICENSE +++ /dev/null @@ -1,669 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ------------------------------------------------------------------------------------- - ------------------------------------------------------------------------------------- - BSD ------------------------------------------------------------------------------------- - -This product bundles asm 3.1, which is available under a "BSD Software License" license. For details, see http://asm.ow2.org/ -This product bundles protobuf-java 2.5.0, which is available under a "BSD Software License" license. For details, see http://code.google.com/p/protobuf -This product bundles jsch 0.1.42, which is available under a "BSD Software License" license. For details, see http://www.jcraft.com/jsch/ -This product bundles paranamer 2.3, which is available under a "BSD Software License" license. For details, see https://github.com/paul-hammant/paranamer -This product bundles leveldbjni-all 1.8, which is available under a "BSD Software License" license. For details, see https://github.com/fusesource/leveldbjni -This product bundles scala-library 2.10.6, which is available under a "BSD Software License" license. For details, see http://www.scala-lang.org/ -This product bundles xmlenc 0.52, which is available under a "BSD Software License" license. For details, see http://xmlenc.sourceforge.net - -Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is -BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ - -This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin -g Package (jaspell): http://jaspell.sourceforge.net/ -License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) - -The KStem stemmer in - analysis/common/src/org/apache/lucene/analysis/en -was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) -under the BSD-license. - -The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default -stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: -analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt -See http://members.unine.ch/jacques.savoy/clef/index.html. - -The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers -(common) are based on BSD-licensed reference implementations created by Jacques Savoy and -Ljiljana Dolamic. These files reside in: -analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java - -The Stempel analyzer (stempel) includes BSD-licensed software developed -by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, -and Edmond Nolan. - -The Polish analyzer (stempel) comes with a default -stopword list that is BSD-licensed created by the Carrot2 project. The file resides -in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. -See http://project.carrot2.org/license.html. - -The Morfologik analyzer (morfologik) includes BSD-licensed software -developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/). - -This product contains a modified version of 'JZlib', a re-implementation of -zlib in pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD Style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product contains a modified version of 'Webbit', a Java event based -WebSocket and HTTP server: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ------------------------------------------------------------------------------------- - CDDL v1.1 ------------------------------------------------------------------------------------- - -This product bundles jersey-guice 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ -This product bundles jersey-client 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ -This product bundles jersey-core 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ -This product bundles jersey-json 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ -This product bundles jersey-server 1.9, which is available under a "Common Development and Distribution License v1.1" license. For details, see https://jersey.java.net/ -This product bundles jaxb-impl 2.2.3-1, which is available under a "Common Development and Distribution License" license. For details, see http://jaxb.java.net/ -This product bundles activation 1.1, which is available under a "Common Development and Distribution License v1.0" license. For details, see http://java.sun.com/products/javabeans/jaf/index.jsp -This product bundles jaxb-api 2.2.2, which is available under a "Common Development and Distribution License" license. For details, see https://jaxb.dev.java.net/ -This product bundles stax-api 1.0-2, which is available under a "Common Development and Distribution License v1.0" license. For details, see https://docs.oracle.com/javase/7/docs/api/javax/xml/stream/package-summary.html - -Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original -source code for this can be found at http://www.eclipse.org/jetty/downloads.php - -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 - 1. Definitions. - 1.1. "Contributor" means each individual or entity that - creates or contributes to the creation of Modifications. - 1.2. "Contributor Version" means the combination of the - Original Software, prior Modifications used by a - Contributor (if any), and the Modifications made by that - particular Contributor. - 1.3. "Covered Software" means (a) the Original Software, or - (b) Modifications, or (c) the combination of files - containing Original Software with files containing - Modifications, in each case including portions thereof. - 1.4. "Executable" means the Covered Software in any form - other than Source Code. - 1.5. "Initial Developer" means the individual or entity - that first makes Original Software available under this - License. - 1.6. "Larger Work" means a work which combines Covered - Software or portions thereof with code not governed by the - terms of this License. - 1.7. "License" means this document. - 1.8. "Licensable" means having the right to grant, to the - maximum extent possible, whether at the time of the initial - grant or subsequently acquired, any and all of the rights - conveyed herein. - 1.9. "Modifications" means the Source Code and Executable - form of any of the following: - A. Any file that results from an addition to, - deletion from or modification of the contents of a - file containing Original Software or previous - Modifications; - B. Any new file that contains any part of the - Original Software or previous Modification; or - C. Any new file that is contributed or otherwise made - available under the terms of this License. - 1.10. "Original Software" means the Source Code and - Executable form of computer software code that is - originally released under this License. - 1.11. "Patent Claims" means any patent claim(s), now owned - or hereafter acquired, including without limitation, - method, process, and apparatus claims, in any patent - Licensable by grantor. - 1.12. "Source Code" means (a) the common form of computer - software code in which modifications are made and (b) - associated documentation included in or with such code. - 1.13. "You" (or "Your") means an individual or a legal - entity exercising rights under, and complying with all of - the terms of, this License. For legal entities, "You" - includes any entity which controls, is controlled by, or is - under common control with You. For purposes of this - definition, "control" means (a) the power, direct or - indirect, to cause the direction or management of such - entity, whether by contract or otherwise, or (b) ownership - of more than fifty percent (50%) of the outstanding shares - or beneficial ownership of such entity. - 2. License Grants. - 2.1. The Initial Developer Grant. - Conditioned upon Your compliance with Section 3.1 below and - subject to third party intellectual property claims, the - Initial Developer hereby grants You a world-wide, - royalty-free, non-exclusive license: - (a) under intellectual property rights (other than - patent or trademark) Licensable by Initial Developer, - to use, reproduce, modify, display, perform, - sublicense and distribute the Original Software (or - portions thereof), with or without Modifications, - and/or as part of a Larger Work; and - (b) under Patent Claims infringed by the making, - using or selling of Original Software, to make, have - made, use, practice, sell, and offer for sale, and/or - otherwise dispose of the Original Software (or - portions thereof). - (c) The licenses granted in Sections 2.1(a) and (b) - are effective on the date Initial Developer first - distributes or otherwise makes the Original Software - available to a third party under the terms of this - License. - (d) Notwithstanding Section 2.1(b) above, no patent - license is granted: (1) for code that You delete from - the Original Software, or (2) for infringements - caused by: (i) the modification of the Original - Software, or (ii) the combination of the Original - Software with other software or devices. - 2.2. Contributor Grant. - Conditioned upon Your compliance with Section 3.1 below and - subject to third party intellectual property claims, each - Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: - (a) under intellectual property rights (other than - patent or trademark) Licensable by Contributor to - use, reproduce, modify, display, perform, sublicense - and distribute the Modifications created by such - Contributor (or portions thereof), either on an - unmodified basis, with other Modifications, as - Covered Software and/or as part of a Larger Work; and - (b) under Patent Claims infringed by the making, - using, or selling of Modifications made by that - Contributor either alone and/or in combination with - its Contributor Version (or portions of such - combination), to make, use, sell, offer for sale, - have made, and/or otherwise dispose of: (1) - Modifications made by that Contributor (or portions - thereof); and (2) the combination of Modifications - made by that Contributor with its Contributor Version - (or portions of such combination). - (c) The licenses granted in Sections 2.2(a) and - 2.2(b) are effective on the date Contributor first - distributes or otherwise makes the Modifications - available to a third party. - (d) Notwithstanding Section 2.2(b) above, no patent - license is granted: (1) for any code that Contributor - has deleted from the Contributor Version; (2) for - infringements caused by: (i) third party - modifications of Contributor Version, or (ii) the - combination of Modifications made by that Contributor - with other software (except as part of the - Contributor Version) or other devices; or (3) under - Patent Claims infringed by Covered Software in the - absence of Modifications made by that Contributor. - 3. Distribution Obligations. - 3.1. Availability of Source Code. - Any Covered Software that You distribute or otherwise make - available in Executable form must also be made available in - Source Code form and that Source Code form must be - distributed only under the terms of this License. You must - include a copy of this License with every copy of the - Source Code form of the Covered Software You distribute or - otherwise make available. You must inform recipients of any - such Covered Software in Executable form as to how they can - obtain such Covered Software in Source Code form in a - reasonable manner on or through a medium customarily used - for software exchange. - 3.2. Modifications. - The Modifications that You create or to which You - contribute are governed by the terms of this License. You - represent that You believe Your Modifications are Your - original creation(s) and/or You have sufficient rights to - grant the rights conveyed by this License. - 3.3. Required Notices. - You must include a notice in each of Your Modifications - that identifies You as the Contributor of the Modification. - You may not remove or alter any copyright, patent or - trademark notices contained within the Covered Software, or - any notices of licensing or any descriptive text giving - attribution to any Contributor or the Initial Developer. - 3.4. Application of Additional Terms. - You may not offer or impose any terms on any Covered - Software in Source Code form that alters or restricts the - applicable version of this License or the recipients' - rights hereunder. You may choose to offer, and to charge a - fee for, warranty, support, indemnity or liability - obligations to one or more recipients of Covered Software. - However, you may do so only on Your own behalf, and not on - behalf of the Initial Developer or any Contributor. You - must make it absolutely clear that any such warranty, - support, indemnity or liability obligation is offered by - You alone, and You hereby agree to indemnify the Initial - Developer and every Contributor for any liability incurred - by the Initial Developer or such Contributor as a result of - warranty, support, indemnity or liability terms You offer. - 3.5. Distribution of Executable Versions. - You may distribute the Executable form of the Covered - Software under the terms of this License or under the terms - of a license of Your choice, which may contain terms - different from this License, provided that You are in - compliance with the terms of this License and that the - license for the Executable form does not attempt to limit - or alter the recipient's rights in the Source Code form - from the rights set forth in this License. If You - distribute the Covered Software in Executable form under a - different license, You must make it absolutely clear that - any terms which differ from this License are offered by You - alone, not by the Initial Developer or Contributor. You - hereby agree to indemnify the Initial Developer and every - Contributor for any liability incurred by the Initial - Developer or such Contributor as a result of any such terms - You offer. - 3.6. Larger Works. - You may create a Larger Work by combining Covered Software - with other code not governed by the terms of this License - and distribute the Larger Work as a single product. In such - a case, You must make sure the requirements of this License - are fulfilled for the Covered Software. - 4. Versions of the License. - 4.1. New Versions. - Sun Microsystems, Inc. is the initial license steward and - may publish revised and/or new versions of this License - from time to time. Each version will be given a - distinguishing version number. Except as provided in - Section 4.3, no one other than the license steward has the - right to modify this License. - 4.2. Effect of New Versions. - You may always continue to use, distribute or otherwise - make the Covered Software available under the terms of the - version of the License under which You originally received - the Covered Software. If the Initial Developer includes a - notice in the Original Software prohibiting it from being - distributed or otherwise made available under any - subsequent version of the License, You must distribute and - make the Covered Software available under the terms of the - version of the License under which You originally received - the Covered Software. Otherwise, You may also choose to - use, distribute or otherwise make the Covered Software - available under the terms of any subsequent version of the - License published by the license steward. - 4.3. Modified Versions. - When You are an Initial Developer and You want to create a - new license for Your Original Software, You may create and - use a modified version of this License if You: (a) rename - the license and remove any references to the name of the - license steward (except to note that the license differs - from this License); and (b) otherwise make it clear that - the license contains terms which differ from this License. - 5. DISCLAIMER OF WARRANTY. - COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" - BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, - INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED - SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR - PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND - PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY - COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE - INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF - ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF - WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF - ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS - DISCLAIMER. - 6. TERMINATION. - 6.1. This License and the rights granted hereunder will - terminate automatically if You fail to comply with terms - herein and fail to cure such breach within 30 days of - becoming aware of the breach. Provisions which, by their - nature, must remain in effect beyond the termination of - this License shall survive. - 6.2. If You assert a patent infringement claim (excluding - declaratory judgment actions) against Initial Developer or - a Contributor (the Initial Developer or Contributor against - whom You assert such claim is referred to as "Participant") - alleging that the Participant Software (meaning the - Contributor Version where the Participant is a Contributor - or the Original Software where the Participant is the - Initial Developer) directly or indirectly infringes any - patent, then any and all rights granted directly or - indirectly to You by such Participant, the Initial - Developer (if the Initial Developer is not the Participant) - and all Contributors under Sections 2.1 and/or 2.2 of this - License shall, upon 60 days notice from Participant - terminate prospectively and automatically at the expiration - of such 60 day notice period, unless if within such 60 day - period You withdraw Your claim with respect to the - Participant Software against such Participant either - unilaterally or pursuant to a written agreement with - Participant. - 6.3. In the event of termination under Sections 6.1 or 6.2 - above, all end user licenses that have been validly granted - by You or any distributor hereunder prior to termination - (excluding licenses granted to You by any distributor) - shall survive termination. - 7. LIMITATION OF LIABILITY. - UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT - (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE - INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF - COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE - LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR - CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT - LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK - STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER - COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN - INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF - LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL - INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT - APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO - NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR - CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT - APPLY TO YOU. - 8. U.S. GOVERNMENT END USERS. - The Covered Software is a "commercial item," as that term is - defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial - computer software" (as that term is defined at 48 C.F.R. ¤ - 252.227-7014(a)(1)) and "commercial computer software - documentation" as such terms are used in 48 C.F.R. 12.212 (Sept. - 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 - through 227.7202-4 (June 1995), all U.S. Government End Users - acquire Covered Software with only those rights set forth herein. - This U.S. Government Rights clause is in lieu of, and supersedes, - any other FAR, DFAR, or other clause or provision that addresses - Government rights in computer software under this License. - 9. MISCELLANEOUS. - This License represents the complete agreement concerning subject - matter hereof. If any provision of this License is held to be - unenforceable, such provision shall be reformed only to the - extent necessary to make it enforceable. This License shall be - governed by the law of the jurisdiction specified in a notice - contained within the Original Software (except to the extent - applicable law, if any, provides otherwise), excluding such - jurisdiction's conflict-of-law provisions. Any litigation - relating to this License shall be subject to the jurisdiction of - the courts located in the jurisdiction and venue specified in a - notice contained within the Original Software, with the losing - party responsible for costs, including, without limitation, court - costs and reasonable attorneys' fees and expenses. The - application of the United Nations Convention on Contracts for the - International Sale of Goods is expressly excluded. Any law or - regulation which provides that the language of a contract shall - be construed against the drafter shall not apply to this License. - You agree that You alone are responsible for compliance with the - United States export administration regulations (and the export - control laws and regulation of any other countries) when You use, - distribute or otherwise make available any Covered Software. - 10. RESPONSIBILITY FOR CLAIMS. - As between Initial Developer and the Contributors, each party is - responsible for claims and damages arising, directly or - indirectly, out of its utilization of rights under this License - and You agree to work with Initial Developer and Contributors to - distribute such responsibility on an equitable basis. Nothing - herein is intended or shall be deemed to constitute any admission - of liability. - ------------------------------------------------------------------------------------- - MIT ------------------------------------------------------------------------------------- - -This product bundles jopt-simple 4.9, which is available under a "MIT Software License" license. For details, see http://jopt-simple.sourceforge.net -This product bundles jcodings 1.0.8, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/jcodings -This product bundles joni 2.1.2, which is available under a "MIT Software License" license. For details, see https://github.com/jruby/joni -This product bundles slf4j-api 1.7.7, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org -This product bundles slf4j-log4j12 1.7.10, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org - -ICU4J, (under analysis/icu) is licensed under an MIT styles license - -The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were -automatically generated with the moman/finenight FSA library, created by -Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, -see http://sites.google.com/site/rrettesite/moman and -http://bitbucket.org/jpbarrette/moman/overview/ - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------------- - Public Domain ------------------------------------------------------------------------------------- - -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - - diff --git a/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/NOTICE b/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/NOTICE deleted file mode 100644 index 09d36cdcce..0000000000 --- a/metron-platform/metron-elasticsearch-test/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,99 +0,0 @@ - -metron-elasticsearch -Copyright 2006-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -ICU4J, (under analysis/icu) -Copyright (c) 1995-2008 International Business Machines Corporation and others - -The SmartChineseAnalyzer source code (smartcn) was -provided by Xiaoping Gao and copyright 2009 by www.imdict.net. - -The BracketFinder (package org.apache.commons.math3.optimization.univariate) -and PowellOptimizer (package org.apache.commons.math3.optimization.general) -classes are based on the Python code in module "optimize.py" (version 0.5) -developed by Travis E. Oliphant for the SciPy library (http://www.scipy.org/) -Copyright © 2003-2009 SciPy Developers. -=============================================================================== - -The LinearConstraint, LinearObjectiveFunction, LinearOptimizer, -RelationShip, SimplexSolver and SimplexTableau classes in package -org.apache.commons.math3.optimization.linear include software developed by -Benjamin McCann (http://www.benmccann.com) and distributed with -the following copyright: Copyright 2009 Google Inc. -=============================================================================== - -This product includes software developed by the -University of Chicago, as Operator of Argonne National -Laboratory. -The LevenbergMarquardtOptimizer class in package -org.apache.commons.math3.optimization.general includes software -translated from the lmder, lmpar and qrsolv Fortran routines -from the Minpack package -Minpack Copyright Notice (1999) University of Chicago. All rights reserved -=============================================================================== - -The GraggBulirschStoerIntegrator class in package -org.apache.commons.math3.ode.nonstiff includes software translated -from the odex Fortran routine developed by E. Hairer and G. Wanner. -Original source copyright: -Copyright (c) 2004, Ernst Hairer -=============================================================================== - -The EigenDecompositionImpl class in package -org.apache.commons.math3.linear includes software translated -from some LAPACK Fortran routines. Original source copyright: -Copyright (c) 1992-2008 The University of Tennessee. All rights reserved. -=============================================================================== - -The MersenneTwister class in package org.apache.commons.math3.random -includes software translated from the 2002-01-26 version of -the Mersenne-Twister generator written in C by Makoto Matsumoto and Takuji -Nishimura. Original source copyright: -Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, -All rights reserved -=============================================================================== - -The LocalizedFormatsTest class in the unit tests is an adapted version of -the OrekitMessagesTest class from the orekit library distributed under the -terms of the Apache 2 licence. Original source copyright: -Copyright 2010 CS Systèmes d'Information -=============================================================================== - -The HermiteInterpolator class and its corresponding test have been imported from -the orekit library distributed under the terms of the Apache 2 licence. Original -source copyright: -Copyright 2010-2012 CS Systèmes d'Information -=============================================================================== - -The creation of the package "o.a.c.m.analysis.integration.gauss" was inspired -by an original code donated by Sébastien Brisard. -=============================================================================== - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2011 The Netty Project - -This product includes software developed for Orekit by -CS Systèmes d'Information (http://www.c-s.fr/) -Copyright 2010-2012 CS Systèmes d'Information - -This project contains annotations derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net - -Objenesis -Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita - -Google Guice - Core Library -Copyright 2006-2011 Google, Inc. - -Google Guice - Extensions - Servlet -Copyright 2006-2011 Google, Inc. - diff --git a/metron-platform/metron-elasticsearch-test/src/main/scripts/start_elasticsearch_topology.sh b/metron-platform/metron-elasticsearch-test/src/main/scripts/start_elasticsearch_topology.sh deleted file mode 100755 index 8ee7518d61..0000000000 --- a/metron-platform/metron-elasticsearch-test/src/main/scripts/start_elasticsearch_topology.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -METRON_VERSION=${project.version} -METRON_HOME=/usr/metron/$METRON_VERSION -TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar -storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties diff --git a/metron-platform/metron-elasticsearch-test/src/test/java/org/apache/metron/elasticsearch/IndexingTest.java b/metron-platform/metron-elasticsearch-test/src/test/java/org/apache/metron/elasticsearch/IndexingTest.java deleted file mode 100644 index 01ea3bc4ec..0000000000 --- a/metron-platform/metron-elasticsearch-test/src/test/java/org/apache/metron/elasticsearch/IndexingTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.elasticsearch; - -import java.io.File; -import java.io.IOException; -import org.apache.commons.io.FileUtils; -import org.apache.http.HttpHost; -import org.apache.metron.test.utils.UnitTestHelper; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeValidationException; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class IndexingTest { - - private RestHighLevelClient client; - private static Node node; - private static final int httpPort = 9200; - - @BeforeClass - public static void oneTimeSetup() throws IOException { - File indexDir = UnitTestHelper.createTempDir(IndexingTest.class.toString()); - File logDir= new File(indexDir, "/logs"); - File dataDir= new File(indexDir, "/data"); - try { - cleanDir(logDir); - cleanDir(dataDir); - - } catch (IOException e) { - throw new RuntimeException("Unable to clean log or data directories", e); - } - - Settings.Builder settingsBuilder = Settings.builder() - .put("cluster.name", "metron") - .put("http.enabled", true) -// .put("http.port", httpPort) - .put("path.logs",logDir.getAbsolutePath()) - .put("path.data",dataDir.getAbsolutePath()) - .put("path.home", indexDir.getAbsoluteFile()) - .put("http.type", "http") - .put("transport.type", "local"); -// .put("index.number_of_shards", 1) -// .put("node.mode", "network") -// .put("index.number_of_replicas", 1); - node = new Node(settingsBuilder.build()); - wait(node, 60000); - } - - private static void cleanDir(File dir) throws IOException { - if(dir.exists()) { - FileUtils.deleteDirectory(dir); - } - dir.mkdirs(); - } - - private static void wait(Node node, long timeoutMillis) { - try { - node.start(); - ClusterHealthResponse chr = (ClusterHealthResponse) node.client() - .execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus( - ClusterHealthStatus.YELLOW).timeout(new TimeValue(timeoutMillis))).actionGet(); - if (chr != null && chr.isTimedOut()) { - throw new RuntimeException("cluster state is " + chr.getStatus().name() - + " and not " + ClusterHealthStatus.YELLOW.name() - + ", from here on, everything will fail!"); - } - } catch (NodeValidationException e) { - throw new RuntimeException("node validation exception"); - } - } - - @Before - public void setup() { -// RestClient restClient = RestClient.builder(new HttpHost("localhost", httpPort, "http")).build(); - RestClient restClient = RestClient.builder(new HttpHost("localhost", httpPort, "http")).build(); - client = new RestHighLevelClient(restClient); - } - - @Test - public void indexes_values() throws IOException { - System.out.println(client.info().getClusterName()); - } - -} diff --git a/metron-platform/metron-elasticsearch-test/src/test/resources/log4j.properties b/metron-platform/metron-elasticsearch-test/src/test/resources/log4j.properties deleted file mode 100644 index 0d50388fff..0000000000 --- a/metron-platform/metron-elasticsearch-test/src/test/resources/log4j.properties +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Root logger option -log4j.rootLogger=ERROR, stdout - -# Direct log messages to stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 000548440e..57475bdee9 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -33,6 +33,12 @@ elasticsearch-shaded ${project.parent.version} + + org.elasticsearch.plugin + transport-netty4-client + ${global_elasticsearch_version} + test + org.apache.metron metron-enrichment @@ -210,6 +216,16 @@ test-jar test + + org.apache.logging.log4j + log4j-api + 2.8.2 + + + org.apache.logging.log4j + log4j-core + 2.8.2 + diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 62706e3a09..be94f40cca 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -45,10 +45,6 @@ import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; -import org.elasticsearch.action.ActionWriteResponse.ShardInfo; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.*; -import org.elasticsearch.action.update.UpdateRequest; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.search.SortOrder; import org.apache.metron.indexing.dao.update.Document; @@ -56,45 +52,29 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.index.mapper.ip.IpFieldMapper; +import org.elasticsearch.index.mapper.LegacyIpFieldMapper; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; -import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder; +import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.*; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import java.io.IOException; -import java.util.Arrays; -import java.util.Date; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; public class ElasticsearchDao implements IndexDao { private transient TransportClient client; @@ -115,7 +95,8 @@ public ElasticsearchDao() { static { Map fieldTypeMap = new HashMap<>(); - fieldTypeMap.put("string", FieldType.STRING); + fieldTypeMap.put("text", FieldType.TEXT); + fieldTypeMap.put("keyword", FieldType.KEYWORD); fieldTypeMap.put("ip", FieldType.IP); fieldTypeMap.put("integer", FieldType.INTEGER); fieldTypeMap.put("long", FieldType.LONG); @@ -149,17 +130,19 @@ protected SearchResponse search(SearchRequest searchRequest, QueryBuilder queryB .size(searchRequest.getSize()) .from(searchRequest.getFrom()) .query(queryBuilder) - .trackScores(true); - searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder())));Optional> fields = searchRequest.getFields(); + + searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); + Optional> fields = searchRequest.getFields(); if (fields.isPresent()) { - searchSourceBuilder.fields(fields.get()); + searchSourceBuilder.storedFields(fields.get()); } else { searchSourceBuilder.fetchSource(true); } Optional> facetFields = searchRequest.getFacetFields(); if (facetFields.isPresent()) { - facetFields.get().forEach(field -> searchSourceBuilder.aggregation(new TermsBuilder(getFacentAggregationName(field)).field(field))); + // https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_bucket_aggregations.html + facetFields.get().forEach(field -> searchSourceBuilder.aggregation(AggregationBuilders.terms(getFacentAggregationName(field)).field(field))); } String[] wildcardIndices = searchRequest.getIndices().stream().map(index -> String.format("%s*", index)).toArray(value -> new String[searchRequest.getIndices().size()]); org.elasticsearch.action.search.SearchResponse elasticsearchResponse; @@ -253,9 +236,7 @@ Optional searchByGuid(String guid, String sensorType, Function> getFacetCounts(List fields, Aggreg } private String formatKey(Object key, FieldType type) { - if (FieldType.IP.equals(type)) { - return IpFieldMapper.longToIp((Long) key); + if (FieldType.IP.equals(type) && key instanceof Long) { + return LegacyIpFieldMapper.longToIp((Long) key); } else if (FieldType.BOOLEAN.equals(type)) { return (Long) key == 1 ? "true" : "false"; } else { @@ -417,11 +398,12 @@ private String formatKey(Object key, FieldType type) { } } - private TermsBuilder getGroupsTermBuilder(GroupRequest groupRequest, int index) { + private TermsAggregationBuilder getGroupsTermBuilder(GroupRequest groupRequest, int index) { List groups = groupRequest.getGroups(); Group group = groups.get(index); String aggregationName = getGroupByAggregationName(group.getField()); - TermsBuilder termsBuilder = new TermsBuilder(aggregationName) + TermsAggregationBuilder termsBuilder = AggregationBuilders.terms(aggregationName); + termsBuilder .field(group.getField()) .size(accessConfig.getMaxSearchGroups()) .order(getElasticsearchGroupOrder(group.getOrder())); @@ -430,7 +412,8 @@ private TermsBuilder getGroupsTermBuilder(GroupRequest groupRequest, int index) } Optional scoreField = groupRequest.getScoreField(); if (scoreField.isPresent()) { - termsBuilder.subAggregation(new SumBuilder(getSumAggregationName(scoreField.get())).field(scoreField.get()).missing(0)); + SumAggregationBuilder scoreSumAggregationBuilder = AggregationBuilders.sum(getSumAggregationName(scoreField.get())).field(scoreField.get()).missing(0); + termsBuilder.subAggregation(scoreSumAggregationBuilder); } return termsBuilder; } diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index 2fb9686687..7c9c1003ba 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import org.apache.lucene.search.join.ScoreMode; import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; @@ -51,19 +52,19 @@ import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.update.Document; -import org.elasticsearch.action.ActionWriteResponse.ShardInfo; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.get.MultiGetRequest.Item; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.index.query.support.QueryInnerHitBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -182,7 +183,8 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx .must(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString())) .must(nestedQuery( ALERT_FIELD, - new QueryStringQueryBuilder(searchRequest.getQuery()) + new QueryStringQueryBuilder(searchRequest.getQuery()), + ScoreMode.None ) ) ) @@ -228,8 +230,9 @@ protected org.elasticsearch.action.search.SearchResponse getMetaAlertsForAlert(S nestedQuery( ALERT_FIELD, boolQuery() - .must(termQuery(ALERT_FIELD + "." + Constants.GUID, guid)) - ).innerHit(new QueryInnerHitBuilder()) + .must(termQuery(ALERT_FIELD + "." + Constants.GUID, guid)), + ScoreMode.None + ).innerHit(new InnerHitBuilder()) ) .must(termQuery(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString())); SearchRequest sr = new SearchRequest(); @@ -239,7 +242,7 @@ protected org.elasticsearch.action.search.SearchResponse getMetaAlertsForAlert(S return elasticsearchDao .getClient() .prepareSearch(index) - .addFields("*") + .addStoredField("*") .setFetchSource(true) .setQuery(qb) .execute() diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java index c7c4d90bb8..ccc7f4cf9c 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java @@ -20,16 +20,20 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.metron.common.configuration.writer.WriterConfiguration; -import org.apache.metron.elasticsearch.writer.ElasticsearchWriter; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.text.SimpleDateFormat; -import java.util.*; +import org.elasticsearch.transport.client.PreBuiltTransportClient; public class ElasticsearchUtils { @@ -55,7 +59,7 @@ public static String getIndexName(String sensorType, String indexPostfix, Writer } public static TransportClient getClient(Map globalConfiguration, Map optionalSettings) { - Settings.Builder settingsBuilder = Settings.settingsBuilder(); + Settings.Builder settingsBuilder = Settings.builder(); settingsBuilder.put("cluster.name", globalConfiguration.get("es.clustername")); settingsBuilder.put("client.transport.ping_timeout","500s"); if (optionalSettings != null) { @@ -64,7 +68,7 @@ public static TransportClient getClient(Map globalConfiguration, Settings settings = settingsBuilder.build(); TransportClient client; try{ - client = TransportClient.builder().settings(settings).build(); + client = new PreBuiltTransportClient(settings); for(HostnamePort hp : getIps(globalConfiguration)) { client.addTransportAddress( new InetSocketTransportAddress(InetAddress.getByName(hp.hostname), hp.port) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java index dd32532b25..96b24826af 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java @@ -17,19 +17,13 @@ */ package org.apache.metron.elasticsearch.writer; -import org.apache.metron.common.Constants; -import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Tuple; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import java.io.Serializable; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; +import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.interfaces.FieldNameConverter; import org.apache.metron.common.writer.BulkMessageWriter; diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java index c28ffc7852..69b0576f81 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java @@ -17,30 +17,36 @@ */ package org.apache.metron.elasticsearch.dao; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import org.apache.metron.elasticsearch.matcher.SearchRequestMatcher; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.search.*; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.search.SortField; +import org.apache.metron.indexing.dao.search.SortOrder; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.mockito.Mock; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - public class ElasticsearchDaoTest { private IndexDao searchService; @@ -106,6 +112,7 @@ public void searchShouldProperlyBuildSearchRequest() throws Exception { public void searchShouldThrowExceptionWhenMaxResultsAreExceeded() throws Exception { SearchRequest searchRequest = new SearchRequest(); searchRequest.setSize(51); + searchRequest.setQuery("test"); try { searchService.search(searchRequest); Assert.fail("Did not throw expected exception"); diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index adb69ee5a3..f35a7de6ad 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.WriteRequest; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -46,7 +47,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * { * "bro_doc": { * "properties": { - * "source:type": { "type": "string" }, + * "source:type": { "type": "keyword" }, * "ip_src_addr": { "type": "ip" }, * "ip_src_port": { "type": "integer" }, * "long_field": { "type": "long" }, @@ -55,8 +56,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "score": { "type": "double" }, * "is_alert": { "type": "boolean" }, * "location_point": { "type": "geo_point" }, - * "bro_field": { "type": "string" }, - * "duplicate_name_field": { "type": "string" } + * "bro_field": { "type": "text", "fielddata": "true" }, + * "duplicate_name_field": { "type": "text", "fielddata": "true" } * } * } * } @@ -68,7 +69,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * { * "snort_doc": { * "properties": { - * "source:type": { "type": "string" }, + * "source:type": { "type": "keyword" }, * "ip_src_addr": { "type": "ip" }, * "ip_src_port": { "type": "integer" }, * "long_field": { "type": "long" }, @@ -86,6 +87,29 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { @Multiline private static String snortTypeMappings; + /** + * { + * "metaalert_doc": { + * "properties": { + * "guid": { "type": "keyword" }, + * "alert": { + * "properties": { + * "guid": { "type": "keyword" } + * } + * }, + * "average": { "type": "keyword" }, + * "min" : { "type": "keyword" }, + * "median" : { "type": "keyword" }, + * "max": { "type": "keyword" }, + * "count": { "type": "keyword" }, + * "sum": { "type": "keyword" } + * } + * } + * } + */ + @Multiline + private static String metaalertTypeMappings; + @Override protected IndexDao createDao() throws Exception { @@ -126,8 +150,10 @@ protected void loadTestData() throws ParseException { .addMapping("bro_doc", broTypeMappings).get(); es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") .addMapping("snort_doc", snortTypeMappings).get(); + es.getClient().admin().indices().prepareCreate("metaalerts") + .addMapping("metaalert_doc", metaalertTypeMappings).get(); - BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefresh(true); + BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); JSONArray broArray = (JSONArray) new JSONParser().parse(broData); for(Object o: broArray) { JSONObject jsonObject = (JSONObject) o; @@ -149,7 +175,6 @@ protected void loadTestData() throws ParseException { JSONObject jsonObject = (JSONObject) o; IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalerts", "metaalert_doc"); indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString()); -// indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString()); bulkRequest.add(indexRequestBuilder); } BulkResponse bulkResponse = bulkRequest.execute().actionGet(); diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java index 171b6ab700..e6c5512f29 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java @@ -17,7 +17,17 @@ */ package org.apache.metron.elasticsearch.integration.components; +import static java.util.Arrays.asList; + import com.fasterxml.jackson.core.type.TypeReference; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; @@ -28,209 +38,230 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; +import org.elasticsearch.transport.Netty4Plugin; public class ElasticSearchComponent implements InMemoryComponent { - public static class Builder{ - private int httpPort; - private File indexDir; - private Map extraElasticSearchSettings = null; - public Builder withHttpPort(int httpPort) { - this.httpPort = httpPort; - return this; - } - public Builder withIndexDir(File indexDir) { - this.indexDir = indexDir; - return this; - } - public Builder withExtraElasticSearchSettings(Map extraElasticSearchSettings) { - this.extraElasticSearchSettings = extraElasticSearchSettings; - return this; - } - public ElasticSearchComponent build() { - return new ElasticSearchComponent(httpPort, indexDir, extraElasticSearchSettings); - } - } + public static class Builder { - private Client client; - private Node node; private int httpPort; private File indexDir; - private Map extraElasticSearchSettings; + private Map extraElasticSearchSettings = null; - public ElasticSearchComponent(int httpPort, File indexDir) { - this(httpPort, indexDir, null); - } - public ElasticSearchComponent(int httpPort, File indexDir, Map extraElasticSearchSettings) { - this.httpPort = httpPort; - this.indexDir = indexDir; - this.extraElasticSearchSettings = extraElasticSearchSettings; + public Builder withHttpPort(int httpPort) { + this.httpPort = httpPort; + return this; } - public Client getClient() { - return client; + + public Builder withIndexDir(File indexDir) { + this.indexDir = indexDir; + return this; } - private void cleanDir(File dir) throws IOException { - if(dir.exists()) { - FileUtils.deleteDirectory(dir); - } - dir.mkdirs(); + public Builder withExtraElasticSearchSettings( + Map extraElasticSearchSettings) { + this.extraElasticSearchSettings = extraElasticSearchSettings; + return this; } - public BulkResponse add(String indexName, String sensorType, String... docs) throws IOException { - List d = new ArrayList<>(); - Collections.addAll(d, docs); - return add(indexName, sensorType, d); + public ElasticSearchComponent build() { + return new ElasticSearchComponent(httpPort, indexDir, extraElasticSearchSettings); } + } + + private static final String STARTUP_TIMEOUT = "60s"; + private Client client; + private Node node; + private int httpPort; + private File indexDir; + private Map extraElasticSearchSettings; + + public ElasticSearchComponent(int httpPort, File indexDir) { + this(httpPort, indexDir, null); + } - public BulkResponse add(String indexName, String sensorType, Iterable docs) throws IOException { - BulkRequestBuilder bulkRequest = getClient().prepareBulk(); - for(String doc : docs) { - IndexRequestBuilder indexRequestBuilder = getClient().prepareIndex(indexName, - sensorType + "_doc"); - - indexRequestBuilder = indexRequestBuilder.setSource(doc); - Map esDoc = JSONUtils.INSTANCE.load(doc, new TypeReference>() { - }); - indexRequestBuilder.setId((String) esDoc.get(Constants.GUID)); - Object ts = esDoc.get("timestamp"); - if(ts != null) { - indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString()); - } - bulkRequest.add(indexRequestBuilder); - } - - BulkResponse response = bulkRequest.execute().actionGet(); - if(response.hasFailures()) { - throw new IOException(response.buildFailureMessage()); - } - return response; + public ElasticSearchComponent(int httpPort, File indexDir, + Map extraElasticSearchSettings) { + this.httpPort = httpPort; + this.indexDir = indexDir; + this.extraElasticSearchSettings = extraElasticSearchSettings; + } + + @Override + public void start() throws UnableToStartException { + File logDir = new File(indexDir, "/logs"); + File dataDir = new File(indexDir, "/data"); + try { + cleanDir(logDir); + cleanDir(dataDir); + } catch (IOException e) { + throw new UnableToStartException("Unable to clean log or data directories", e); } - public void createIndexWithMapping(String indexName, String mappingType, String mappingSource) - throws IOException { - CreateIndexResponse cir = client.admin().indices().prepareCreate(indexName) - .addMapping(mappingType, mappingSource) - .get(); + Settings.Builder settingsBuilder = Settings.builder() + .put("cluster.name", "metron") + .put("path.logs",logDir.getAbsolutePath()) + .put("path.data",dataDir.getAbsolutePath()) + .put("path.home", indexDir.getAbsoluteFile()) + .put("transport.type", "netty4") + .put("http.enabled", "false"); - if (!cir.isAcknowledged()) { - throw new IOException("Create index was not acknowledged"); - } + if (extraElasticSearchSettings != null) { + settingsBuilder = settingsBuilder.put(extraElasticSearchSettings); } - @Override - public void start() throws UnableToStartException { - File logDir= new File(indexDir, "/logs"); - File dataDir= new File(indexDir, "/data"); - try { - cleanDir(logDir); - cleanDir(dataDir); + node = new TestNode(settingsBuilder.build(), asList(Netty4Plugin.class)); + client = node.client(); + try { + node.start(); + } catch (NodeValidationException e) { + throw new UnableToStartException("Error starting ES node.", e); + } + waitForCluster(client, ClusterHealthStatus.YELLOW, STARTUP_TIMEOUT); + } - } catch (IOException e) { - throw new UnableToStartException("Unable to clean log or data directories", e); - } + private void cleanDir(File dir) throws IOException { + if (dir.exists()) { + FileUtils.deleteDirectory(dir); + } + dir.mkdirs(); + } - Settings.Builder settingsBuilder = Settings.settingsBuilder() - .put("node.http.enabled", true) - .put("http.port", httpPort) - .put("path.logs",logDir.getAbsolutePath()) - .put("path.data",dataDir.getAbsolutePath()) - .put("path.home", indexDir.getAbsoluteFile()) - .put("index.number_of_shards", 1) - .put("node.mode", "network") - .put("index.number_of_replicas", 1); + // ES 5.x+ needs this to startup a node without using their test framework + private static class TestNode extends Node { - if(extraElasticSearchSettings != null) { + private TestNode(Settings preparedSettings, + Collection> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), classpathPlugins); + } - settingsBuilder = settingsBuilder.put(extraElasticSearchSettings); + } - } + public static void waitForCluster(Client client, ClusterHealthStatus statusThreshold, + String timeout) throws UnableToStartException { + try { + ClusterHealthResponse healthResponse = (ClusterHealthResponse) client + .execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(statusThreshold).timeout(timeout)) + .actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new UnableToStartException("cluster state is " + healthResponse.getStatus().name() + + " and not " + statusThreshold.name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new UnableToStartException( + "timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); + } + } - node = NodeBuilder.nodeBuilder().settings(settingsBuilder).clusterName("metron").node(); - node.start(); + public Client getClient() { + return client; + } - client = node.client(); + public BulkResponse add(String indexName, String sensorType, String... docs) throws IOException { + List d = new ArrayList<>(); + Collections.addAll(d, docs); + return add(indexName, sensorType, d); + } - waitForCluster(client, ClusterHealthStatus.YELLOW, new TimeValue(60000)); + public BulkResponse add(String indexName, String sensorType, Iterable docs) + throws IOException { + BulkRequestBuilder bulkRequest = getClient().prepareBulk(); + for (String doc : docs) { + IndexRequestBuilder indexRequestBuilder = getClient() + .prepareIndex(indexName, sensorType + "_doc"); + indexRequestBuilder = indexRequestBuilder.setSource(doc); + Map esDoc = JSONUtils.INSTANCE + .load(doc, new TypeReference>() { + }); + indexRequestBuilder.setId((String) esDoc.get(Constants.GUID)); + Object ts = esDoc.get("timestamp"); + if (ts != null) { + indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString()); + } + bulkRequest.add(indexRequestBuilder); } - public static void waitForCluster(ElasticsearchClient client, ClusterHealthStatus status, TimeValue timeout) throws UnableToStartException { - try { - ClusterHealthResponse healthResponse = - (ClusterHealthResponse)client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus(status).timeout(timeout)).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new UnableToStartException("cluster state is " + healthResponse.getStatus().name() - + " and not " + status.name() - + ", from here on, everything will fail!"); - } - } catch (ElasticsearchTimeoutException e) { - throw new UnableToStartException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); - } + BulkResponse response = bulkRequest.execute().actionGet(); + if (response.hasFailures()) { + throw new IOException(response.buildFailureMessage()); } + return response; + } - public List> getAllIndexedDocs(String index, String sourceType) throws IOException { - return getAllIndexedDocs(index, sourceType, null); - } - public List> getAllIndexedDocs(String index, String sourceType, String subMessage) throws IOException { - getClient().admin().indices().refresh(new RefreshRequest()); - SearchResponse response = getClient().prepareSearch(index) - .setTypes(sourceType) - .setSource("message") - .setFrom(0) - .setSize(1000) - .execute().actionGet(); - List> ret = new ArrayList>(); - for (SearchHit hit : response.getHits()) { - Object o = null; - if(subMessage == null) { - o = hit.getSource(); - } - else { - o = hit.getSource().get(subMessage); - } - ret.add((Map)(o)); - } - return ret; + public void createIndexWithMapping(String indexName, String mappingType, String mappingSource) + throws IOException { + CreateIndexResponse cir = client.admin().indices().prepareCreate(indexName) + .addMapping(mappingType, mappingSource) + .get(); + + if (!cir.isAcknowledged()) { + throw new IOException("Create index was not acknowledged"); } - public boolean hasIndex(String indexName) { - Set indices = getClient().admin() - .indices() - .stats(new IndicesStatsRequest()) - .actionGet() - .getIndices() - .keySet(); - return indices.contains(indexName); + } + + public List> getAllIndexedDocs(String index, String sourceType) + throws IOException { + return getAllIndexedDocs(index, sourceType, null); + } + public List> getAllIndexedDocs(String index, String sourceType, + String subMessage) throws IOException { + getClient().admin().indices().refresh(new RefreshRequest()); + SearchResponse response = getClient().prepareSearch(index) + .setTypes(sourceType) +// .setSource("message") ?? + .setFrom(0) + .setSize(1000) + .execute().actionGet(); + List> ret = new ArrayList>(); + for (SearchHit hit : response.getHits()) { + Object o = null; + if (subMessage == null) { + o = hit.getSource(); + } else { + o = hit.getSource().get(subMessage); + } + ret.add((Map) (o)); } + return ret; + } + + public boolean hasIndex(String indexName) { + Set indices = getClient().admin() + .indices() + .stats(new IndicesStatsRequest()) + .actionGet() + .getIndices() + .keySet(); + return indices.contains(indexName); + + } @Override public void stop() { + try { node.close(); - node = null; - client = null; + } catch (IOException e) { + throw new RuntimeException("Unable to stop node." , e); + } + node = null; + client = null; } } diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java index 9d694717e0..0ab6307579 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java @@ -19,20 +19,17 @@ import org.apache.metron.indexing.dao.search.SortField; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; +import org.junit.Assert; import org.mockito.ArgumentMatcher; -import java.util.Arrays; - public class SearchRequestMatcher extends ArgumentMatcher { private String[] expectedIndicies; - private BytesReference expectedSource; + private SearchSourceBuilder expectedSource; public SearchRequestMatcher(String[] indices, String query, int size, int from, SortField[] sortFields) { expectedIndicies = indices; @@ -47,14 +44,14 @@ public SearchRequestMatcher(String[] indices, String query, int size, int from, fieldSortBuilder.order(sortField.getSortOrder() == org.apache.metron.indexing.dao.search.SortOrder.DESC ? SortOrder.DESC : SortOrder.ASC); searchSourceBuilder = searchSourceBuilder.sort(fieldSortBuilder); } - expectedSource = searchSourceBuilder.buildAsBytes(Requests.CONTENT_TYPE); + expectedSource = searchSourceBuilder; } @Override public boolean matches(Object o) { SearchRequest searchRequest = (SearchRequest) o; - boolean indiciesMatch = Arrays.equals(expectedIndicies, searchRequest.indices()); - boolean sourcesMatch = searchRequest.source().equals(expectedSource); - return indiciesMatch && sourcesMatch; + Assert.assertArrayEquals("Indices did not match", expectedIndicies, searchRequest.indices()); + Assert.assertEquals("Source did not match", expectedSource, searchRequest.source()); + return true; } } diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java index 1f00cf589b..2abd997932 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java @@ -20,8 +20,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; public enum FieldType { - @JsonProperty("string") - STRING("string"), + @JsonProperty("text") + TEXT("text"), + @JsonProperty("keyword") + KEYWORD("keyword"), @JsonProperty("ip") IP("ip"), @JsonProperty("integer") diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 26d1a75336..15fd7d59c6 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -17,28 +17,26 @@ */ package org.apache.metron.indexing.dao; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GroupRequest; import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.GroupResult; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; -import org.apache.metron.indexing.dao.search.GroupResult; import org.apache.metron.integration.InMemoryComponent; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.junit.*; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; public abstract class SearchIntegrationTest { /** @@ -354,69 +352,139 @@ public synchronized void setup() throws Exception { } @Test - public void test() throws Exception { + public void all_query_returns_all_results() throws Exception { //All Query Testcase - { - SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List results = response.getResults(); - for(int i = 0;i < 5;++i) { - Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); - Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); - } - for(int i = 5;i < 10;++i) { - Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); - Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); - } + SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List results = response.getResults(); + for (int i = 0; i < 5; ++i) { + Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); + Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp")); } - //Filter test case - { - SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(3, response.getTotal()); - List results = response.getResults(); - Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); - Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); - Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); - Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); - Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); - Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); + for (int i = 5; i < 10; ++i) { + Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); + Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp")); } - //Sort test case - { - SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List results = response.getResults(); - for(int i = 8001;i < 8011;++i) { - Assert.assertEquals(i, results.get(i-8001).getSource().get("ip_src_port")); - } + } + + + @Test + public void filter_query_filters_results() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(3, response.getTotal()); + List results = response.getResults(); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); + Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); + } + + @Test + public void sort_query_sorts_results_ascending() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List results = response.getResults(); + for (int i = 8001; i < 8011; ++i) { + Assert.assertEquals(i, results.get(i - 8001).getSource().get("ip_src_port")); } - //pagination test case - { - SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List results = response.getResults(); - Assert.assertEquals(3, results.size()); - Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); - Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); - Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); - Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); - Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); - Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); + } + + @Test + public void results_are_paginated() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List results = response.getResults(); + Assert.assertEquals(3, results.size()); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); + Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); + } + + @Test + public void returns_results_only_for_specified_indices() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(5, response.getTotal()); + List results = response.getResults(); + for (int i = 5, j = 0; i > 0; i--, j++) { + Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); + Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); } + } + + @Test + public void test() throws Exception { + //All Query Testcase +// { +// SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); +// SearchResponse response = dao.search(request); +// Assert.assertEquals(10, response.getTotal()); +// List results = response.getResults(); +// for(int i = 0;i < 5;++i) { +// Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); +// Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); +// } +// for(int i = 5;i < 10;++i) { +// Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); +// Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); +// } +// } + //Filter test case +// { +// SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); +// SearchResponse response = dao.search(request); +// Assert.assertEquals(3, response.getTotal()); +// List results = response.getResults(); +// Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); +// Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); +// Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); +// Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); +// Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); +// Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); +// } + //Sort test case +// { +// searchrequest request = jsonutils.instance.load(sortquery, searchrequest.class); +// searchresponse response = dao.search(request); +// assert.assertequals(10, response.gettotal()); +// list results = response.getresults(); +// for (int i = 8001; i < 8011; ++i) { +// assert.assertequals(i, results.get(i - 8001).getsource().get("ip_src_port")); +// } +// } + //pagination test case +// { +// SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); +// SearchResponse response = dao.search(request); +// Assert.assertEquals(10, response.getTotal()); +// List results = response.getResults(); +// Assert.assertEquals(3, results.size()); +// Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); +// Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); +// Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); +// Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); +// Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); +// Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); +// } //Index query { - SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(5, response.getTotal()); - List results = response.getResults(); - for(int i = 5,j=0;i > 0;i--,j++) { - Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); - Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); - } +// SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); +// SearchResponse response = dao.search(request); +// Assert.assertEquals(5, response.getTotal()); +// List results = response.getResults(); +// for(int i = 5,j=0;i > 0;i--,j++) { +// Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); +// Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); +// } } //Facet query including all field types { @@ -526,7 +594,7 @@ public void test() throws Exception { Assert.assertEquals(2, fieldTypes.size()); Map broTypes = fieldTypes.get("bro"); Assert.assertEquals(12, broTypes.size()); - Assert.assertEquals(FieldType.STRING, broTypes.get("source:type")); + Assert.assertEquals(FieldType.KEYWORD, broTypes.get("source:type")); Assert.assertEquals(FieldType.IP, broTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, broTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, broTypes.get("long_field")); @@ -535,12 +603,12 @@ public void test() throws Exception { Assert.assertEquals(FieldType.DOUBLE, broTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, broTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point")); - Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field")); - Assert.assertEquals(FieldType.STRING, broTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.STRING, broTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("guid")); Map snortTypes = fieldTypes.get("snort"); Assert.assertEquals(12, snortTypes.size()); - Assert.assertEquals(FieldType.STRING, snortTypes.get("source:type")); + Assert.assertEquals(FieldType.KEYWORD, snortTypes.get("source:type")); Assert.assertEquals(FieldType.IP, snortTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, snortTypes.get("long_field")); @@ -551,7 +619,7 @@ public void test() throws Exception { Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.STRING, broTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("guid")); } // getColumnMetadata with only bro { @@ -559,7 +627,7 @@ public void test() throws Exception { Assert.assertEquals(1, fieldTypes.size()); Map broTypes = fieldTypes.get("bro"); Assert.assertEquals(12, broTypes.size()); - Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("bro_field")); } // getColumnMetadata with only snort { @@ -574,7 +642,7 @@ public void test() throws Exception { Map fieldTypes = dao.getCommonColumnMetadata(Arrays.asList("bro", "snort")); // Should only return fields in both Assert.assertEquals(10, fieldTypes.size()); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); @@ -583,14 +651,14 @@ public void test() throws Exception { Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid")); } // getCommonColumnMetadata with only bro { Map fieldTypes = dao.getCommonColumnMetadata(Collections.singletonList("bro")); Assert.assertEquals(12, fieldTypes.size()); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); } // getCommonColumnMetadata with only snort { @@ -600,34 +668,34 @@ public void test() throws Exception { Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); } //Fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List results = response.getResults(); - for(int i = 0;i < 5;++i) { - Map source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertNotNull(source.get("ip_src_addr")); - } - for(int i = 5;i < 10;++i) { - Map source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertNotNull(source.get("ip_src_addr")); - } - } +// { +// SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); +// SearchResponse response = dao.search(request); +// Assert.assertEquals(10, response.getTotal()); +// List results = response.getResults(); +// for(int i = 0;i < 5;++i) { +// Map source = results.get(i).getSource(); +// Assert.assertEquals(1, source.size()); +// Assert.assertNotNull(source.get("ip_src_addr")); +// } +// for(int i = 5;i < 10;++i) { +// Map source = results.get(i).getSource(); +// Assert.assertEquals(1, source.size()); +// Assert.assertNotNull(source.get("ip_src_addr")); +// } +// } //Meta Alerts Fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(2, response.getTotal()); - List results = response.getResults(); - for (int i = 0;i < 2;++i) { - Map source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertEquals(source.get("guid"), "meta_" + (i + 1)); - } - } +// { +// SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class); +// SearchResponse response = dao.search(request); +// Assert.assertEquals(2, response.getTotal()); +// List results = response.getResults(); +// for (int i = 0;i < 2;++i) { +// Map source = results.get(i).getSource(); +// Assert.assertEquals(1, source.size()); +// Assert.assertEquals(source.get("guid"), "meta_" + (i + 1)); +// } +// } //No results fields query { SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); @@ -775,6 +843,37 @@ public void test() throws Exception { } } + @Test + public void queries_fields() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List results = response.getResults(); + for (int i = 0; i < 5; ++i) { + Map source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); + } + for (int i = 5; i < 10; ++i) { + Map source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); + } + } + + @Test + public void searches_metaalerts_fields() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(2, response.getTotal()); + List results = response.getResults(); + for (int i = 0; i < 2; ++i) { + Map source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertEquals(source.get("guid"), "meta_" + (i + 1)); + } + } + @AfterClass public static void stop() throws Exception { indexComponent.stop(); diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml index eeb379ef75..93ced81ca9 100644 --- a/metron-platform/pom.xml +++ b/metron-platform/pom.xml @@ -57,9 +57,7 @@ metron-writer metron-hbase elasticsearch-shaded - elasticsearch-shaded562 metron-elasticsearch - metron-elasticsearch-test metron-storm-kafka metron-storm-kafka-override diff --git a/pom.xml b/pom.xml index 1e142e5e3d..ea32fbf0bb 100644 --- a/pom.xml +++ b/pom.xml @@ -93,8 +93,8 @@ ${base_hadoop_version} ${base_hbase_version} ${base_flume_version} - - 2.3.3 + 5.6.2 + 1.1.1 3.0.2 4.12 @@ -116,7 +116,6 @@ 0.38 0.9.10 8.0 - From 084db5ab4258a1dacb792ba504c1aeb18838327d Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 17 Oct 2017 06:30:23 -0600 Subject: [PATCH 04/59] fix licensing --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea32fbf0bb..f09e2c07b5 100644 --- a/pom.xml +++ b/pom.xml @@ -323,7 +323,7 @@ **/target/** **/bro-plugin-kafka/build/** - **/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p + **/src/main/resources/common-services/KIBANA/**/package/scripts/dashboard/dashboard.p **/packer-build/scripts/** **/packer-build/bin/** From b33575c0b2131b7f44141342ebaaf3664fe6eab9 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 17 Oct 2017 14:23:20 -0600 Subject: [PATCH 05/59] fix elasticsearchdao tests --- .../SearchControllerIntegrationTest.java | 14 +- metron-platform/metron-elasticsearch/pom.xml | 8 +- .../elasticsearch/dao/ElasticsearchDao.java | 23 +- .../ElasticsearchSearchIntegrationTest.java | 11 +- .../indexing/dao/SearchIntegrationTest.java | 662 ++++++++---------- 5 files changed, 323 insertions(+), 395 deletions(-) diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java index d7cf2b0d18..72732c146c 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java @@ -196,11 +196,11 @@ public void test() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(2))) - .andExpect(jsonPath("$.bro.common_string_field").value("string")) + .andExpect(jsonPath("$.bro.common_string_field").value("text")) .andExpect(jsonPath("$.bro.common_integer_field").value("integer")) .andExpect(jsonPath("$.bro.bro_field").value("boolean")) .andExpect(jsonPath("$.bro.duplicate_field").value("date")) - .andExpect(jsonPath("$.snort.common_string_field").value("string")) + .andExpect(jsonPath("$.snort.common_string_field").value("text")) .andExpect(jsonPath("$.snort.common_integer_field").value("integer")) .andExpect(jsonPath("$.snort.snort_field").value("double")) .andExpect(jsonPath("$.snort.duplicate_field").value("long")); @@ -209,14 +209,14 @@ public void test() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(2))) - .andExpect(jsonPath("$.common_string_field").value("string")) + .andExpect(jsonPath("$.common_string_field").value("text")) .andExpect(jsonPath("$.common_integer_field").value("integer")); this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\"]")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(1))) - .andExpect(jsonPath("$.bro.common_string_field").value("string")) + .andExpect(jsonPath("$.bro.common_string_field").value("text")) .andExpect(jsonPath("$.bro.common_integer_field").value("integer")) .andExpect(jsonPath("$.bro.bro_field").value("boolean")) .andExpect(jsonPath("$.bro.duplicate_field").value("date")); @@ -225,7 +225,7 @@ public void test() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(4))) - .andExpect(jsonPath("$.common_string_field").value("string")) + .andExpect(jsonPath("$.common_string_field").value("text")) .andExpect(jsonPath("$.common_integer_field").value("integer")) .andExpect(jsonPath("$.bro_field").value("boolean")) .andExpect(jsonPath("$.duplicate_field").value("date")); @@ -234,7 +234,7 @@ public void test() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(1))) - .andExpect(jsonPath("$.snort.common_string_field").value("string")) + .andExpect(jsonPath("$.snort.common_string_field").value("text")) .andExpect(jsonPath("$.snort.common_integer_field").value("integer")) .andExpect(jsonPath("$.snort.snort_field").value("double")) .andExpect(jsonPath("$.snort.duplicate_field").value("long")); @@ -243,7 +243,7 @@ public void test() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(4))) - .andExpect(jsonPath("$.common_string_field").value("string")) + .andExpect(jsonPath("$.common_string_field").value("text")) .andExpect(jsonPath("$.common_integer_field").value("integer")) .andExpect(jsonPath("$.snort_field").value("double")) .andExpect(jsonPath("$.duplicate_field").value("long")); diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 57475bdee9..953c95dc5e 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -191,9 +191,15 @@ + + org.hamcrest + hamcrest-core + 1.3 + test + org.mockito - mockito-all + mockito-core ${global_mockito_version} test diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index be94f40cca..62e526c835 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -130,19 +130,23 @@ protected SearchResponse search(SearchRequest searchRequest, QueryBuilder queryB .size(searchRequest.getSize()) .from(searchRequest.getFrom()) .query(queryBuilder) +// .fetchSource(true) .trackScores(true); searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); Optional> fields = searchRequest.getFields(); if (fields.isPresent()) { - searchSourceBuilder.storedFields(fields.get()); +// searchSourceBuilder.storedFields(fields.get()); +// searchSourceBuilder.fetchSource(fields.get().toArray(new String[]{}), null); + searchSourceBuilder.fetchSource("*", null); } else { searchSourceBuilder.fetchSource(true); } Optional> facetFields = searchRequest.getFacetFields(); if (facetFields.isPresent()) { // https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_bucket_aggregations.html - facetFields.get().forEach(field -> searchSourceBuilder.aggregation(AggregationBuilders.terms(getFacentAggregationName(field)).field(field))); + facetFields.get().forEach(field -> searchSourceBuilder.aggregation(AggregationBuilders.terms( + getFacetAggregationName(field)).field(field))); } String[] wildcardIndices = searchRequest.getIndices().stream().map(index -> String.format("%s*", index)).toArray(value -> new String[searchRequest.getIndices().size()]); org.elasticsearch.action.search.SearchResponse elasticsearchResponse; @@ -155,7 +159,7 @@ protected SearchResponse search(SearchRequest searchRequest, QueryBuilder queryB SearchResponse searchResponse = new SearchResponse(); searchResponse.setTotal(elasticsearchResponse.getHits().getTotalHits()); searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit -> - getSearchResult(searchHit, fields.isPresent())).collect(Collectors.toList())); + getSearchResult(searchHit, fields)).collect(Collectors.toList())); if (facetFields.isPresent()) { Map commonColumnMetadata; try { @@ -378,7 +382,7 @@ public Map> getFacetCounts(List fields, Aggreg Map> fieldCounts = new HashMap<>(); for (String field: fields) { Map valueCounts = new HashMap<>(); - Aggregation aggregation = aggregations.get(getFacentAggregationName(field)); + Aggregation aggregation = aggregations.get(getFacetAggregationName(field)); if (aggregation instanceof Terms) { Terms terms = (Terms) aggregation; terms.getBuckets().stream().forEach(bucket -> valueCounts.put(formatKey(bucket.getKey(), commonColumnMetadata.get(field)), bucket.getDocCount())); @@ -441,14 +445,15 @@ private List getGroupResults(GroupRequest groupRequest, int index, return searchResultGroups; } - private SearchResult getSearchResult(SearchHit searchHit, boolean fieldsPresent) { + private SearchResult getSearchResult(SearchHit searchHit, Optional> fields) { SearchResult searchResult = new SearchResult(); searchResult.setId(searchHit.getId()); Map source; - if (fieldsPresent) { + if (fields.isPresent()) { + Map resultSourceAsMap = searchHit.getSourceAsMap(); source = new HashMap<>(); - searchHit.getFields().forEach((key, value) -> { - source.put(key, value.getValues().size() == 1 ? value.getValue() : value.getValues()); + fields.get().forEach(field -> { + source.put(field, resultSourceAsMap.get(field)); }); } else { source = searchHit.getSource(); @@ -459,7 +464,7 @@ private SearchResult getSearchResult(SearchHit searchHit, boolean fieldsPresent) return searchResult; } - private String getFacentAggregationName(String field) { + private String getFacetAggregationName(String field) { return String.format("%s_count", field); } diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index f35a7de6ad..4ed00f2fe2 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -18,6 +18,8 @@ package org.apache.metron.elasticsearch.integration; +import java.io.File; +import java.util.HashMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao; @@ -36,9 +38,6 @@ import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import java.io.File; -import java.util.HashMap; - public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { private static String indexDir = "target/elasticsearch_search"; private static String dateFormat = "yyyy.MM.dd.HH"; @@ -93,6 +92,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "properties": { * "guid": { "type": "keyword" }, * "alert": { + * "type": "nested", * "properties": { * "guid": { "type": "keyword" } * } @@ -110,7 +110,6 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { @Multiline private static String metaalertTypeMappings; - @Override protected IndexDao createDao() throws Exception { IndexDao elasticsearchDao = new ElasticsearchDao(); @@ -181,5 +180,9 @@ protected void loadTestData() throws ParseException { if (bulkResponse.hasFailures()) { throw new RuntimeException("Failed to index test data"); } +// SearchRequestBuilder metaalerts = es.getClient().prepareSearch("metaalerts") +// .setQuery(QueryBuilders.matchAllQuery()).setFetchSource(true); +// SearchResponse response = metaalerts.get(); +// System.out.println("blah"); } } diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 15fd7d59c6..44d774ac98 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -36,7 +36,9 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public abstract class SearchIntegrationTest { /** @@ -351,6 +353,9 @@ public synchronized void setup() throws Exception { } } + @Rule + public ExpectedException thrown = ExpectedException.none(); + @Test public void all_query_returns_all_results() throws Exception { //All Query Testcase @@ -368,7 +373,6 @@ public void all_query_returns_all_results() throws Exception { } } - @Test public void filter_query_filters_results() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); @@ -422,205 +426,139 @@ public void returns_results_only_for_specified_indices() throws Exception { } @Test - public void test() throws Exception { - //All Query Testcase -// { -// SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); -// SearchResponse response = dao.search(request); -// Assert.assertEquals(10, response.getTotal()); -// List results = response.getResults(); -// for(int i = 0;i < 5;++i) { -// Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); -// Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); -// } -// for(int i = 5;i < 10;++i) { -// Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); -// Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); -// } -// } - //Filter test case -// { -// SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); -// SearchResponse response = dao.search(request); -// Assert.assertEquals(3, response.getTotal()); -// List results = response.getResults(); -// Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); -// Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); -// Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); -// Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); -// Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); -// Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); -// } - //Sort test case -// { -// searchrequest request = jsonutils.instance.load(sortquery, searchrequest.class); -// searchresponse response = dao.search(request); -// assert.assertequals(10, response.gettotal()); -// list results = response.getresults(); -// for (int i = 8001; i < 8011; ++i) { -// assert.assertequals(i, results.get(i - 8001).getsource().get("ip_src_port")); -// } -// } - //pagination test case -// { -// SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); -// SearchResponse response = dao.search(request); -// Assert.assertEquals(10, response.getTotal()); -// List results = response.getResults(); -// Assert.assertEquals(3, results.size()); -// Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); -// Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); -// Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); -// Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); -// Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); -// Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); -// } - //Index query - { -// SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); -// SearchResponse response = dao.search(request); -// Assert.assertEquals(5, response.getTotal()); -// List results = response.getResults(); -// for(int i = 5,j=0;i > 0;i--,j++) { -// Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); -// Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); -// } - } - //Facet query including all field types - { - SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - Map> facetCounts = response.getFacetCounts(); - Assert.assertEquals(8, facetCounts.size()); - Map sourceTypeCounts = facetCounts.get("source:type"); - Assert.assertEquals(2, sourceTypeCounts.size()); - Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro")); - Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort")); - Map ipSrcAddrCounts = facetCounts.get("ip_src_addr"); - Assert.assertEquals(8, ipSrcAddrCounts.size()); - Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8")); - Map ipSrcPortCounts = facetCounts.get("ip_src_port"); - Assert.assertEquals(10, ipSrcPortCounts.size()); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010")); - Map longFieldCounts = facetCounts.get("long_field"); - Assert.assertEquals(2, longFieldCounts.size()); - Assert.assertEquals(new Long(8), longFieldCounts.get("10000")); - Assert.assertEquals(new Long(2), longFieldCounts.get("20000")); - Map timestampCounts = facetCounts.get("timestamp"); - Assert.assertEquals(10, timestampCounts.size()); - Assert.assertEquals(new Long(1), timestampCounts.get("1")); - Assert.assertEquals(new Long(1), timestampCounts.get("2")); - Assert.assertEquals(new Long(1), timestampCounts.get("3")); - Assert.assertEquals(new Long(1), timestampCounts.get("4")); - Assert.assertEquals(new Long(1), timestampCounts.get("5")); - Assert.assertEquals(new Long(1), timestampCounts.get("6")); - Assert.assertEquals(new Long(1), timestampCounts.get("7")); - Assert.assertEquals(new Long(1), timestampCounts.get("8")); - Assert.assertEquals(new Long(1), timestampCounts.get("9")); - Assert.assertEquals(new Long(1), timestampCounts.get("10")); - Map latitudeCounts = facetCounts.get("latitude"); - Assert.assertEquals(2, latitudeCounts.size()); - List latitudeKeys = new ArrayList<>(latitudeCounts.keySet()); - Collections.sort(latitudeKeys); - Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001); - Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001); - Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0))); - Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1))); - Map scoreFieldCounts = facetCounts.get("score"); - Assert.assertEquals(4, scoreFieldCounts.size()); - List scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet()); - Collections.sort(scoreFieldKeys); - Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001); - Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001); - Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001); - Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001); - Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0))); - Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1))); - Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2))); - Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3))); - Map isAlertCounts = facetCounts.get("is_alert"); - Assert.assertEquals(2, isAlertCounts.size()); - Assert.assertEquals(new Long(6), isAlertCounts.get("true")); - Assert.assertEquals(new Long(4), isAlertCounts.get("false")); - } - //Bad facet query - { - SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class); - try { - dao.search(request); - Assert.fail("Exception expected, but did not come."); - } - catch(InvalidSearchException ise) { - Assert.assertEquals("Could not execute search", ise.getMessage()); - } - } - //Disabled facet query - { - SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertNull(response.getFacetCounts()); - } - //Exceeded maximum results query - { - SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); - try { - dao.search(request); - Assert.fail("Exception expected, but did not come."); - } - catch(InvalidSearchException ise) { - Assert.assertEquals("Search result size must be less than 100", ise.getMessage()); - } - } - // getColumnMetadata with multiple indices - { - Map> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); - Assert.assertEquals(2, fieldTypes.size()); - Map broTypes = fieldTypes.get("bro"); - Assert.assertEquals(12, broTypes.size()); - Assert.assertEquals(FieldType.KEYWORD, broTypes.get("source:type")); - Assert.assertEquals(FieldType.IP, broTypes.get("ip_src_addr")); - Assert.assertEquals(FieldType.INTEGER, broTypes.get("ip_src_port")); - Assert.assertEquals(FieldType.LONG, broTypes.get("long_field")); - Assert.assertEquals(FieldType.DATE, broTypes.get("timestamp")); - Assert.assertEquals(FieldType.FLOAT, broTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, broTypes.get("score")); - Assert.assertEquals(FieldType.BOOLEAN, broTypes.get("is_alert")); - Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point")); - Assert.assertEquals(FieldType.TEXT, broTypes.get("bro_field")); - Assert.assertEquals(FieldType.TEXT, broTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.TEXT, broTypes.get("guid")); - Map snortTypes = fieldTypes.get("snort"); - Assert.assertEquals(12, snortTypes.size()); - Assert.assertEquals(FieldType.KEYWORD, snortTypes.get("source:type")); - Assert.assertEquals(FieldType.IP, snortTypes.get("ip_src_addr")); - Assert.assertEquals(FieldType.INTEGER, snortTypes.get("ip_src_port")); - Assert.assertEquals(FieldType.LONG, snortTypes.get("long_field")); - Assert.assertEquals(FieldType.DATE, snortTypes.get("timestamp")); - Assert.assertEquals(FieldType.FLOAT, snortTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, snortTypes.get("score")); - Assert.assertEquals(FieldType.BOOLEAN, snortTypes.get("is_alert")); - Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point")); - Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); - Assert.assertEquals(FieldType.INTEGER, snortTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.TEXT, broTypes.get("guid")); - } + public void facet_query_yields_field_types() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + Map> facetCounts = response.getFacetCounts(); + Assert.assertEquals(8, facetCounts.size()); + Map sourceTypeCounts = facetCounts.get("source:type"); + Assert.assertEquals(2, sourceTypeCounts.size()); + Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro")); + Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort")); + Map ipSrcAddrCounts = facetCounts.get("ip_src_addr"); + Assert.assertEquals(8, ipSrcAddrCounts.size()); + Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8")); + Map ipSrcPortCounts = facetCounts.get("ip_src_port"); + Assert.assertEquals(10, ipSrcPortCounts.size()); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010")); + Map longFieldCounts = facetCounts.get("long_field"); + Assert.assertEquals(2, longFieldCounts.size()); + Assert.assertEquals(new Long(8), longFieldCounts.get("10000")); + Assert.assertEquals(new Long(2), longFieldCounts.get("20000")); + Map timestampCounts = facetCounts.get("timestamp"); + Assert.assertEquals(10, timestampCounts.size()); + Assert.assertEquals(new Long(1), timestampCounts.get("1")); + Assert.assertEquals(new Long(1), timestampCounts.get("2")); + Assert.assertEquals(new Long(1), timestampCounts.get("3")); + Assert.assertEquals(new Long(1), timestampCounts.get("4")); + Assert.assertEquals(new Long(1), timestampCounts.get("5")); + Assert.assertEquals(new Long(1), timestampCounts.get("6")); + Assert.assertEquals(new Long(1), timestampCounts.get("7")); + Assert.assertEquals(new Long(1), timestampCounts.get("8")); + Assert.assertEquals(new Long(1), timestampCounts.get("9")); + Assert.assertEquals(new Long(1), timestampCounts.get("10")); + Map latitudeCounts = facetCounts.get("latitude"); + Assert.assertEquals(2, latitudeCounts.size()); + List latitudeKeys = new ArrayList<>(latitudeCounts.keySet()); + Collections.sort(latitudeKeys); + Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001); + Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001); + Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0))); + Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1))); + Map scoreFieldCounts = facetCounts.get("score"); + Assert.assertEquals(4, scoreFieldCounts.size()); + List scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet()); + Collections.sort(scoreFieldKeys); + Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001); + Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001); + Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001); + Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001); + Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0))); + Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1))); + Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2))); + Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3))); + Map isAlertCounts = facetCounts.get("is_alert"); + Assert.assertEquals(2, isAlertCounts.size()); + Assert.assertEquals(new Long(6), isAlertCounts.get("true")); + Assert.assertEquals(new Long(4), isAlertCounts.get("false")); + } + + @Test + public void bad_facet_query_throws_exception() throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Could not execute search"); + SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class); + dao.search(request); + } + + @Test + public void disabled_facet_query_returns_null_count() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertNull(response.getFacetCounts()); + } + + @Test + public void exceeding_max_resulsts_throws_exception() throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Search result size must be less than 100"); + SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); + dao.search(request); + } + + @Test + public void returns_column_data_for_multiple_indices() throws Exception { + Map> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); + Assert.assertEquals(2, fieldTypes.size()); + Map broTypes = fieldTypes.get("bro"); + Assert.assertEquals(12, broTypes.size()); + Assert.assertEquals(FieldType.KEYWORD, broTypes.get("source:type")); + Assert.assertEquals(FieldType.IP, broTypes.get("ip_src_addr")); + Assert.assertEquals(FieldType.INTEGER, broTypes.get("ip_src_port")); + Assert.assertEquals(FieldType.LONG, broTypes.get("long_field")); + Assert.assertEquals(FieldType.DATE, broTypes.get("timestamp")); + Assert.assertEquals(FieldType.FLOAT, broTypes.get("latitude")); + Assert.assertEquals(FieldType.DOUBLE, broTypes.get("score")); + Assert.assertEquals(FieldType.BOOLEAN, broTypes.get("is_alert")); + Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("guid")); + Map snortTypes = fieldTypes.get("snort"); + Assert.assertEquals(12, snortTypes.size()); + Assert.assertEquals(FieldType.KEYWORD, snortTypes.get("source:type")); + Assert.assertEquals(FieldType.IP, snortTypes.get("ip_src_addr")); + Assert.assertEquals(FieldType.INTEGER, snortTypes.get("ip_src_port")); + Assert.assertEquals(FieldType.LONG, snortTypes.get("long_field")); + Assert.assertEquals(FieldType.DATE, snortTypes.get("timestamp")); + Assert.assertEquals(FieldType.FLOAT, snortTypes.get("latitude")); + Assert.assertEquals(FieldType.DOUBLE, snortTypes.get("score")); + Assert.assertEquals(FieldType.BOOLEAN, snortTypes.get("is_alert")); + Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point")); + Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); + Assert.assertEquals(FieldType.INTEGER, snortTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.TEXT, broTypes.get("guid")); + } + + @Test + public void returns_column_metadata_for_specified_indices() throws Exception { // getColumnMetadata with only bro { Map> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); @@ -653,6 +591,10 @@ public void test() throws Exception { Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid")); } + } + + @Test + public void returns_common_metadata_for_specified_indices() throws Exception { // getCommonColumnMetadata with only bro { Map fieldTypes = dao.getCommonColumnMetadata(Collections.singletonList("bro")); @@ -667,180 +609,152 @@ public void test() throws Exception { Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); } - //Fields query -// { -// SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); -// SearchResponse response = dao.search(request); -// Assert.assertEquals(10, response.getTotal()); -// List results = response.getResults(); -// for(int i = 0;i < 5;++i) { -// Map source = results.get(i).getSource(); -// Assert.assertEquals(1, source.size()); -// Assert.assertNotNull(source.get("ip_src_addr")); -// } -// for(int i = 5;i < 10;++i) { -// Map source = results.get(i).getSource(); -// Assert.assertEquals(1, source.size()); -// Assert.assertNotNull(source.get("ip_src_addr")); -// } -// } - //Meta Alerts Fields query -// { -// SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class); -// SearchResponse response = dao.search(request); -// Assert.assertEquals(2, response.getTotal()); -// List results = response.getResults(); -// for (int i = 0;i < 2;++i) { -// Map source = results.get(i).getSource(); -// Assert.assertEquals(1, source.size()); -// Assert.assertEquals(source.get("guid"), "meta_" + (i + 1)); -// } -// } - //No results fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(0, response.getTotal()); - } - // Group by test case, default order is count descending - { - GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class); - GroupResponse response = dao.group(request); - Assert.assertEquals("is_alert", response.getGroupedBy()); - List isAlertGroups = response.getGroupResults(); - Assert.assertEquals(2, isAlertGroups.size()); - - // isAlert == true group - GroupResult trueGroup = isAlertGroups.get(0); - Assert.assertEquals("true", trueGroup.getKey()); - Assert.assertEquals(6, trueGroup.getTotal()); - Assert.assertEquals("latitude", trueGroup.getGroupedBy()); - Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001); - List trueLatitudeGroups = trueGroup.getGroupResults(); - Assert.assertEquals(2, trueLatitudeGroups.size()); - - - // isAlert == true && latitude == 48.5839 group - GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); - Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); - Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001); - - // isAlert == true && latitude == 48.0001 group - GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1); - Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); - Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001); - - // isAlert == false group - GroupResult falseGroup = isAlertGroups.get(1); - Assert.assertEquals("false", falseGroup.getKey()); - Assert.assertEquals("latitude", falseGroup.getGroupedBy()); - Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001); - List falseLatitudeGroups = falseGroup.getGroupResults(); - Assert.assertEquals(2, falseLatitudeGroups.size()); - - // isAlert == false && latitude == 48.5839 group - GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0); - Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); - Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001); - - // isAlert == false && latitude == 48.0001 group - GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1); - Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); - Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001); - } - // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending - { - GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class); - GroupResponse response = dao.group(request); - Assert.assertEquals("is_alert", response.getGroupedBy()); - List isAlertGroups = response.getGroupResults(); - Assert.assertEquals(2, isAlertGroups.size()); - - // isAlert == false group - GroupResult falseGroup = isAlertGroups.get(0); - Assert.assertEquals(4, falseGroup.getTotal()); - Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy()); - List falseIpSrcAddrGroups = falseGroup.getGroupResults(); - Assert.assertEquals(4, falseIpSrcAddrGroups.size()); - - // isAlert == false && ip_src_addr == 192.168.1.8 group - GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0); - Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.7 group - GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1); - Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.6 group - GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2); - Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.2 group - GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3); - Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults()); - - // isAlert == false group - GroupResult trueGroup = isAlertGroups.get(1); - Assert.assertEquals(6, trueGroup.getTotal()); - Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy()); - List trueIpSrcAddrGroups = trueGroup.getGroupResults(); - Assert.assertEquals(4, trueIpSrcAddrGroups.size()); - - // isAlert == false && ip_src_addr == 192.168.1.5 group - GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0); - Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.4 group - GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1); - Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.3 group - GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2); - Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.1 group - GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3); - Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey()); - Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults()); - } - //Bad group query - { - GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class); - try { - dao.group(request); - Assert.fail("Exception expected, but did not come."); - } - catch(InvalidSearchException ise) { - Assert.assertEquals("Could not execute search", ise.getMessage()); - } - } + } + + @Test + public void no_results_returned_when_query_does_not_match() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(0, response.getTotal()); + } + + @Test + public void group_by_returns_results_in_groups() throws Exception { + GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List isAlertGroups = response.getGroupResults(); + Assert.assertEquals(2, isAlertGroups.size()); + + // isAlert == true group + GroupResult trueGroup = isAlertGroups.get(0); + Assert.assertEquals("true", trueGroup.getKey()); + Assert.assertEquals(6, trueGroup.getTotal()); + Assert.assertEquals("latitude", trueGroup.getGroupedBy()); + Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001); + List trueLatitudeGroups = trueGroup.getGroupResults(); + Assert.assertEquals(2, trueLatitudeGroups.size()); + + + // isAlert == true && latitude == 48.5839 group + GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); + Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); + Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001); + + // isAlert == true && latitude == 48.0001 group + GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1); + Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); + Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001); + + // isAlert == false group + GroupResult falseGroup = isAlertGroups.get(1); + Assert.assertEquals("false", falseGroup.getKey()); + Assert.assertEquals("latitude", falseGroup.getGroupedBy()); + Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001); + List falseLatitudeGroups = falseGroup.getGroupResults(); + Assert.assertEquals(2, falseLatitudeGroups.size()); + + // isAlert == false && latitude == 48.5839 group + GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0); + Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); + Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001); + + // isAlert == false && latitude == 48.0001 group + GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1); + Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); + Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001); + } + + @Test + public void group_by_returns_results_in_sorted_groups() throws Exception { + GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List isAlertGroups = response.getGroupResults(); + Assert.assertEquals(2, isAlertGroups.size()); + + // isAlert == false group + GroupResult falseGroup = isAlertGroups.get(0); + Assert.assertEquals(4, falseGroup.getTotal()); + Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy()); + List falseIpSrcAddrGroups = falseGroup.getGroupResults(); + Assert.assertEquals(4, falseIpSrcAddrGroups.size()); + + // isAlert == false && ip_src_addr == 192.168.1.8 group + GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0); + Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.7 group + GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1); + Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.6 group + GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2); + Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.2 group + GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3); + Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults()); + + // isAlert == false group + GroupResult trueGroup = isAlertGroups.get(1); + Assert.assertEquals(6, trueGroup.getTotal()); + Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy()); + List trueIpSrcAddrGroups = trueGroup.getGroupResults(); + Assert.assertEquals(4, trueIpSrcAddrGroups.size()); + + // isAlert == false && ip_src_addr == 192.168.1.5 group + GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0); + Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.4 group + GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1); + Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.3 group + GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2); + Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.1 group + GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3); + Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey()); + Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults()); + } + + @Test + public void throws_exception_on_aggregation_queries_on_non_string_non_numeric_fields() + throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Could not execute search"); + GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class); + dao.group(request); } @Test From 132b3330260dfe5a4b97e91d6210b381eda7222d Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 18 Oct 2017 19:17:32 -0600 Subject: [PATCH 06/59] working through ES setting migrations for 2.x to 5.x --- .../5.6.2/configuration/elastic-site.xml | 9 ++------- .../5.6.2/configuration/elastic-sysconfig.xml | 6 ++---- .../5.6.2/package/scripts/params.py | 3 +-- .../templates/elasticsearch.master.yaml.j2 | 15 +++++---------- .../templates/elasticsearch.slave.yaml.j2 | 17 ++++++----------- 5 files changed, 16 insertions(+), 34 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml index 7369f921a9..34df1e49ab 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml @@ -75,11 +75,6 @@ Set a custom port for the node to node communication - - discovery_zen_ping_multicast_enabled - false - Whether to use multicast - discovery_zen_ping_timeout 3s @@ -143,7 +138,7 @@ Percentage of heap used for write buffers - bootstrap_mlockall + bootstrap_memory_lock true The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out @@ -194,7 +189,7 @@ network_publish_host - + [] true diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml index 44ecf05ac4..2f37537539 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml @@ -63,9 +63,6 @@ # Directory where the Elasticsearch binary distribution resides ES_HOME={{elastic_home}} -# Heap Size (defaults to 256m min, 1g max) -ES_HEAP_SIZE={{heap_size}} - # Maximum number of open files MAX_OPEN_FILES={{max_open_files}} @@ -98,7 +95,8 @@ JAVA_HOME={{java64_home}} # Additional Java OPTS ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled \ -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \ --XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8" +-XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8 \ +-Xms{{heap_size}} -Xmx{{heap_size}}" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py index e7b8d85c0f..2596759a9e 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py @@ -53,7 +53,6 @@ recover_after_time = config['configurations']['elastic-site']['recover_after_time'] gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes'] expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes'] -discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled'] index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count'] index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size'] index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval'] @@ -61,7 +60,7 @@ index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards'] index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas'] indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size'] -bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall'] +bootstrap_memory_lock = config['configurations']['elastic-site']['bootstrap_memory_lock'] threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size'] cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries'] cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 index f0d2a8f30d..da59770e03 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 @@ -27,8 +27,6 @@ cluster: discovery: zen: ping: - multicast: - enabled: {{ discovery_zen_ping_multicast_enabled }} unicast: hosts: {{zen_discovery_ping_unicast_hosts}} @@ -52,13 +50,10 @@ gateway: recover_after_time: {{recover_after_time}} expected_data_nodes: {{expected_data_nodes}} -index: - number_of_shards: {{index_number_of_shards}} - merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} - translog.flush_threshold_size: {{index_translog_flush_threshold_size}} - refresh_interval: {{index_refresh_interval}} - number_of_replicas: {{index_number_of_replicas}} - +# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html +transient: + indices.store.throttle.type: merge + indices: memory: index_buffer_size: {{indices_memory_index_buffer_size}} @@ -68,7 +63,7 @@ indices: cluster: send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} -bootstrap.mlockall: {{bootstrap_mlockall}} +bootstrap.memory_lock: {{bootstrap_memory_lock}} threadpool: bulk: diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 index 7d2d0cf8a0..dc2071326d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 @@ -27,8 +27,6 @@ cluster: discovery: zen: ping: - multicast: - enabled: {{discovery_zen_ping_multicast_enabled}} unicast: hosts: {{zen_discovery_ping_unicast_hosts}} @@ -51,14 +49,11 @@ gateway: recover_after_data_nodes: {{gateway_recover_after_data_nodes}} recover_after_time: {{recover_after_time}} expected_data_nodes: {{expected_data_nodes}} - -index: - number_of_shards: {{index_number_of_shards}} - merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} - translog.flush_threshold_size: {{index_translog_flush_threshold_size}} - refresh_interval: {{index_refresh_interval}} - number_of_replicas: {{index_number_of_replicas}} - + +# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html +transient: + indices.store.throttle.type: merge + indices: memory: index_buffer_size: {{indices_memory_index_buffer_size}} @@ -68,7 +63,7 @@ indices: cluster: send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} -bootstrap.mlockall: {{bootstrap_mlockall}} +bootstrap.memory_lock: {{bootstrap_memory_lock}} threadpool: bulk: From f71321485ea7b54be4ba45e969991bf3696c5197 Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 19 Oct 2017 16:07:47 -0400 Subject: [PATCH 07/59] Updating to fix kibana mpack and log4j issue. --- .../KIBANA/5.6.2/package/scripts/kibana_master.py | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 12 ++++++++++++ metron-platform/metron-elasticsearch/pom.xml | 2 +- pom.xml | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py index faca7a83fc..87d678886c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py @@ -48,7 +48,7 @@ def configure(self, env, upgrade_type=None, config_dir=None): directories = [params.log_dir, params.pid_dir, params.conf_dir] Directory(directories, - # recursive=True, + create_parents=True, mode=0755, owner=params.kibana_user, group=params.kibana_user diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index 0e3cbe420c..0a26b5c4cf 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -73,6 +73,18 @@ com.fasterxml.jackson.core jackson-core + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 953c95dc5e..5de1d863b4 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -230,7 +230,7 @@ org.apache.logging.log4j log4j-core - 2.8.2 + ${global_log4j_core_version} diff --git a/pom.xml b/pom.xml index f09e2c07b5..b9849be64b 100644 --- a/pom.xml +++ b/pom.xml @@ -116,6 +116,7 @@ 0.38 0.9.10 8.0 + 2.1 From 54c2486e4e618a6d445667ed2040faacf165ca8c Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 19 Oct 2017 16:29:19 -0400 Subject: [PATCH 08/59] Removing argline..we should figure out why this is necessary, though. I tried to up the version of surefire. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b9849be64b..374349452c 100644 --- a/pom.xml +++ b/pom.xml @@ -156,7 +156,7 @@ ${global_surefire_version} - @{argLine} -Xmx2048m + -Xmx2048m true false From bd48988cf61344eeb2eebdd61caacf1ce83be1d5 Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 19 Oct 2017 19:02:23 -0400 Subject: [PATCH 09/59] Avoiding log4j version mismatch. --- metron-platform/elasticsearch-shaded/pom.xml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index 0a26b5c4cf..ac5c2bb9a0 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -86,8 +86,13 @@ log4j - - + + + org.apache.logging.log4j + log4j-core + 2.8.2 + + org.elasticsearch elasticsearch ${global_elasticsearch_version} @@ -161,7 +166,11 @@ - + + + org.apache.logging.log4j + org.apache.metron.logging.log4j + com.google.common org.apache.metron.guava.elasticsearch-shaded From 785e6a08558dec9e38468bc7ed50253f4e97ecdd Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 20 Oct 2017 12:43:01 -0400 Subject: [PATCH 10/59] Fixed some parameter name changes. --- .../ELASTICSEARCH/5.6.2/package/scripts/params.py | 12 +++++++++--- .../package/templates/elasticsearch.master.yaml.j2 | 2 +- .../package/templates/elasticsearch.slave.yaml.j2 | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py index 2596759a9e..b63c71b830 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py @@ -20,6 +20,12 @@ from resource_management.libraries.script import Script +def yamlify_variables(var) : + if isinstance(var, type(True)): + return str(var).lower() + else: + return var + # server configurations config = Script.get_config() @@ -60,14 +66,14 @@ index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards'] index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas'] indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size'] -bootstrap_memory_lock = config['configurations']['elastic-site']['bootstrap_memory_lock'] +bootstrap_memory_lock = yamlify_variables(config['configurations']['elastic-site']['bootstrap_memory_lock']) threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size'] cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries'] cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low'] -cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled'] +cluster_routing_allocation_disk_threshold_enabled = yamlify_variables(config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled']) cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high'] indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size'] -indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping'] +indices_cluster_send_refresh_mapping = yamlify_variables(config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping']) threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size'] discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 index da59770e03..b645782b6a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 @@ -65,7 +65,7 @@ indices: bootstrap.memory_lock: {{bootstrap_memory_lock}} -threadpool: +thread_pool: bulk: queue_size: {{threadpool_bulk_queue_size}} index: diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 index dc2071326d..ff4dda8503 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 @@ -65,7 +65,7 @@ indices: bootstrap.memory_lock: {{bootstrap_memory_lock}} -threadpool: +thread_pool: bulk: queue_size: {{threadpool_bulk_queue_size}} index: From b8c81273de0fd82fa4c2978e8e62b3819173aaa7 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 20 Oct 2017 14:33:14 -0400 Subject: [PATCH 11/59] Removing and relocating old properties. --- .../5.6.2/package/templates/elasticsearch.master.yaml.j2 | 9 ++++----- .../5.6.2/package/templates/elasticsearch.slave.yaml.j2 | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 index b645782b6a..d3f6c6de39 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 @@ -37,9 +37,10 @@ node: path: data: {{path_data}} -http.cors.enabled: {{http_cors_enabled}} +http: + port: {{http_port}} + cors.enabled: {{http_cors_enabled}} -port: {{http_port}} transport: tcp: @@ -55,13 +56,11 @@ transient: indices.store.throttle.type: merge indices: + store.throttle.type: {{indices_memory_index_store_throttle_type}} memory: index_buffer_size: {{indices_memory_index_buffer_size}} - store.throttle.type: {{indices_memory_index_store_throttle_type}} fielddata: cache.size: {{indices_fielddata_cache_size}} - cluster: - send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} bootstrap.memory_lock: {{bootstrap_memory_lock}} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 index ff4dda8503..341d808b86 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 @@ -37,9 +37,10 @@ node: path: data: {{path_data}} -http.cors.enabled: {{http_cors_enabled}} +http: + port: {{http_port}} + cors.enabled: {{http_cors_enabled}} -port: {{http_port}} transport: tcp: @@ -55,13 +56,11 @@ transient: indices.store.throttle.type: merge indices: + store.throttle.type: {{indices_memory_index_store_throttle_type}} memory: index_buffer_size: {{indices_memory_index_buffer_size}} - store.throttle.type: {{indices_memory_index_store_throttle_type}} fielddata: cache.size: {{indices_fielddata_cache_size}} - cluster: - send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} bootstrap.memory_lock: {{bootstrap_memory_lock}} From dcb2dbd1a96a31501fe4a0e0ffbdac70bc4e18b2 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 20 Oct 2017 17:38:28 -0400 Subject: [PATCH 12/59] Updating yamls. --- .../5.6.2/package/templates/elasticsearch.master.yaml.j2 | 4 ---- .../5.6.2/package/templates/elasticsearch.slave.yaml.j2 | 3 --- 2 files changed, 7 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 index d3f6c6de39..2fa1dacac6 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 @@ -50,11 +50,7 @@ gateway: recover_after_data_nodes: {{gateway_recover_after_data_nodes}} recover_after_time: {{recover_after_time}} expected_data_nodes: {{expected_data_nodes}} - # https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html -transient: - indices.store.throttle.type: merge - indices: store.throttle.type: {{indices_memory_index_store_throttle_type}} memory: diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 index 341d808b86..04c8c08da0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 @@ -52,9 +52,6 @@ gateway: expected_data_nodes: {{expected_data_nodes}} # https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html -transient: - indices.store.throttle.type: merge - indices: store.throttle.type: {{indices_memory_index_store_throttle_type}} memory: From d5629f0f53207012ee78c862d23c100431f8d2b1 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 20 Oct 2017 22:30:44 -0400 Subject: [PATCH 13/59] updating yaml. --- .../ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml | 2 +- .../5.6.2/package/templates/elasticsearch.slave.yaml.j2 | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml index 2f37537539..6779a1765c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml @@ -46,7 +46,7 @@ max_open_files - 65535 + 65536 Maximum number of open files diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 index 04c8c08da0..6bf8399d31 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 @@ -59,7 +59,9 @@ indices: fielddata: cache.size: {{indices_fielddata_cache_size}} -bootstrap.memory_lock: {{bootstrap_memory_lock}} +bootstrap: + memory_lock: {{bootstrap_memory_lock}} + system_call_filter: false thread_pool: bulk: From d5e5cf7f9c353c45ce49b5fd47b2d37acf55d7c3 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 20 Oct 2017 22:51:11 -0400 Subject: [PATCH 14/59] Added a limits role to adjust security. We should consider doing this in the mpack possibly..not sure how... --- .../roles/ambari_common/meta/main.yml | 1 + metron-deployment/roles/limits/tasks/main.yml | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 metron-deployment/roles/limits/tasks/main.yml diff --git a/metron-deployment/roles/ambari_common/meta/main.yml b/metron-deployment/roles/ambari_common/meta/main.yml index d7e46d6696..6077b5af67 100644 --- a/metron-deployment/roles/ambari_common/meta/main.yml +++ b/metron-deployment/roles/ambari_common/meta/main.yml @@ -19,3 +19,4 @@ dependencies: - libselinux-python - epel - ntp + - limits diff --git a/metron-deployment/roles/limits/tasks/main.yml b/metron-deployment/roles/limits/tasks/main.yml new file mode 100644 index 0000000000..bb590d1e46 --- /dev/null +++ b/metron-deployment/roles/limits/tasks/main.yml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Adjust nproc security limits for ES + pam_limits: + domain: elasticsearch + limit_type: '-' + limit_item: noproc + value: 2048 + +- name: Adjust nofile security limits for ES + pam_limits: + domain: elasticsearch + limit_type: '-' + limit_item: nofile + value: 65536 + +- name: Adjust memlock soft security limits for ES + pam_limits: + domain: elasticsearch + limit_type: soft + limit_item: memlock + value: unlimited + +- name: Adjust memlock hard security limits for ES + pam_limits: + domain: elasticsearch + limit_type: hard + limit_item: memlock + value: unlimited + From 67cc97d922ec06f231e45852dfc71a57618432c2 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 20 Oct 2017 23:20:46 -0400 Subject: [PATCH 15/59] typo --- metron-deployment/roles/limits/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/roles/limits/tasks/main.yml b/metron-deployment/roles/limits/tasks/main.yml index bb590d1e46..27e0ad0587 100644 --- a/metron-deployment/roles/limits/tasks/main.yml +++ b/metron-deployment/roles/limits/tasks/main.yml @@ -19,7 +19,7 @@ pam_limits: domain: elasticsearch limit_type: '-' - limit_item: noproc + limit_item: nproc value: 2048 - name: Adjust nofile security limits for ES From 297ed1d938892f5de48cc66caab013262be8cb4d Mon Sep 17 00:00:00 2001 From: cstella Date: Sat, 21 Oct 2017 00:22:41 -0400 Subject: [PATCH 16/59] forgot to remove bootstrap filter checking from master --- .../5.6.2/package/templates/elasticsearch.master.yaml.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 index 2fa1dacac6..8e20ba2b26 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 @@ -58,7 +58,9 @@ indices: fielddata: cache.size: {{indices_fielddata_cache_size}} -bootstrap.memory_lock: {{bootstrap_memory_lock}} +bootstrap: + memory_lock: {{bootstrap_memory_lock}} + system_call_filter: false thread_pool: bulk: From fc92aa9edaacdfa3c618411a9f3f2426fcf51f1f Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Sun, 22 Oct 2017 21:55:32 -0600 Subject: [PATCH 17/59] Get ES templates working --- .../CURRENT/package/files/bro_index.template | 467 +++++++++--------- .../package/files/error_index.template | 40 +- .../CURRENT/package/files/meta_index.mapping | 19 +- .../package/files/snort_index.template | 71 +-- .../CURRENT/package/files/yaf_index.template | 12 +- 5 files changed, 314 insertions(+), 295 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template index 7db006ebaa..fa7bc9a578 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template @@ -2,9 +2,6 @@ "template": "bro_index*", "mappings": { "bro_doc": { - "_timestamp": { - "enabled": true - }, "dynamic_templates": [ { "geo_location_point": { @@ -20,8 +17,8 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -30,8 +27,8 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -40,8 +37,8 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -50,8 +47,8 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -60,8 +57,8 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -105,7 +102,8 @@ { "threat_triage_reason": { "mapping": { - "type": "string" + "type": "text", + "fielddata": "true" }, "match": "threat.triage.rules:*:reason", "match_mapping_type": "*" @@ -114,7 +112,8 @@ { "threat_triage_name": { "mapping": { - "type": "string" + "type": "text", + "fielddata": "true" }, "match": "threat.triage.rules:*:name", "match_mapping_type": "*" @@ -137,8 +136,8 @@ * Metron-specific fields */ "source:type": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * Widely-used Bro fields (potentially renamed during Metron ingest) @@ -148,8 +147,8 @@ "format": "epoch_millis" }, "uid": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "alert": { "type": "nested" @@ -196,28 +195,28 @@ "type": "integer" }, "method": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "host": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "uri": { - "type": "string", - "index": "not_analyzed", - "ignore_above": 8191 + "type": "text", + "index": "false" }, "referrer": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "version": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "user_agent": { - "type": "string" + "type": "text", + "fielddata": "true" }, "request_body_len": { "type": "long" @@ -229,16 +228,16 @@ "type": "integer" }, "status_msg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "username": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "password": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "capture_password": { "type": "boolean" @@ -255,36 +254,36 @@ * Notes: Field exists in the DNS and DHCP logs */ "proto": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "trans_id": { "type": "long" }, "query": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "qclass": { "type": "integer" }, "qclass_name": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "qtype": { "type": "integer" }, "qtype_name": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "rcode": { "type": "integer" }, "rcode_name": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "AA": { "type": "boolean" @@ -302,7 +301,8 @@ "type": "integer" }, "answers": { - "type": "string" + "type": "text", + "fielddata": "true" }, "rejected": { "type": "boolean" @@ -322,58 +322,58 @@ * Notes: Field exists in the Conn and Files logs */ "service": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "duration": { "type": "float" }, "orig_bytes": { "type": "long", - "index": "not_analyzed" + "index": "false" }, "resp_bytes": { "type": "long", - "index": "not_analyzed" + "index": "false" }, "conn_state": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "local_orig": { "type": "boolean" }, "local_resp": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "missed_bytes": { "type": "long", - "index": "not_analyzed" + "index": "false" }, "history": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "orig_pkts": { "type": "long", - "index": "not_analyzed" + "index": "false" }, "orig_ip_bytes": { "type": "long", - "index": "not_analyzed" + "index": "false" }, "resp_pkts": { "type": "long", - "index": "not_analyzed" + "index": "false" }, "resp_ip_bytes": { "type": "long", - "index": "not_analyzed" + "index": "false" }, "tunnel_parents": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * DPD log support @@ -384,12 +384,12 @@ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs */ "analyzer": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "failure_reason": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * FTP log support @@ -409,19 +409,21 @@ * Notes: Field exists in the FTP and Notice logs */ "user": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "command": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "arg": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "mime_type": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "file_size": { @@ -431,8 +433,8 @@ "type": "integer" }, "reply_msg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "data_channel:passive": { "type": "boolean" @@ -447,15 +449,16 @@ "type": "integer" }, "cwd": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "passive": { "type": "boolean" }, "fuid": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * Files log support @@ -472,23 +475,25 @@ * Notes: Field exists in the FTP and Files logs */ "conn_uids": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "source": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "depth": { "type": "integer" }, "analyzers": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "filename": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "is_orig": { "type": "boolean" @@ -509,20 +514,20 @@ "type": "boolean" }, "parent_fuid": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "md5": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "sha1": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "sha256": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * Known::CertInfo log support @@ -536,16 +541,18 @@ "type": "integer" }, "subject": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "issuer_subject": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "serial": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * SMTP log support @@ -556,66 +563,75 @@ * Notes: Field exists in the Known::CertInfo and SMTP logs */ "helo": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "mailfrom": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "rcptto": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "date": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "from": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "to": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "reply_to": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "msg_id": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "in_reply_to": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "x_originating_ip": { "type": "ip" }, "first_received": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "second_received": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "last_reply": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "path": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "tls": { "type": "boolean" }, "fuids": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "is_webmail": { "type": "boolean" @@ -629,27 +645,27 @@ * Notes: Field exists in the HTTP, SSL, and SSH logs */ "cipher": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "curve": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "server_name": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "resumed": { "type": "boolean" }, "last_alert": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "next_protocol": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "established": { "type": "boolean" @@ -659,19 +675,19 @@ * https://www.bro.org/sphinx/scripts/base/frameworks/notice/weird.bro.html#type-Weird::Info */ "name": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "addl": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "notice": { "type": "boolean" }, "peer": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * Notice log support @@ -685,24 +701,24 @@ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs */ "file_mime_type": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "file_desc": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "note": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "msg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "sub": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "src": { "type": "ip" @@ -712,26 +728,26 @@ }, "p": { "type": "integer", - "index": "not_analyzed" + "index": "false" }, "n": { "type": "integer", - "index": "not_analyzed" + "index": "false" }, "src_peer": { "type": "ip" }, "peer_descr": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "actions": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "suppress_for": { "type": "double", - "index": "not_analyzed" + "index": "false" }, "dropped": { "type": "boolean" @@ -748,15 +764,15 @@ * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs */ "mac": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "assigned_ip": { "type": "ip" }, "lease_time": { "type": "float", - "index": "not_analyzed" + "index": "false" }, /* * SSH log support @@ -771,43 +787,43 @@ }, "auth_attempts": { "type": "integer", - "index": "not_analyzed" + "index": "false" }, "direction": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "client": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "server": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "cipher_alg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "mac_alg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "compression_alg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "kex_alg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "host_key_alg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "host_key": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * Software log support @@ -819,34 +835,35 @@ */ "host_p": { "type": "integer", - "index": "not_analyzed" + "index": "false" }, "software_type": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "version:major": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "version:minor": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "version:minor2": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "version:minor3": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "version:addl": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "unparsed_version": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, /* @@ -864,12 +881,12 @@ "type": "ip" }, "connect_info": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "result": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, /* * X509 log support @@ -883,79 +900,79 @@ * logs, however, id is a string to identify the certificate file id. */ "id": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:version": { "type": "integer", - "index": "not_analyzed" + "index": "false" }, "certificate:serial": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:subject": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:issuer": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:not_valid_before": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:not_valid_after": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:key_alg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:sig_alg": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:key_type": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:key_length": { "type": "integer", - "index": "not_analyzed" + "index": "false" }, "certificate:exponent": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "certificate:curve": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "san:dns": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "san:uri": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "san:email": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "san:ip": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "basic_constraints:ca": { "type": "boolean" }, "basic_constraints:path_len": { "type": "integer", - "index": "not_analyzed" + "index": "false" }, /* * Known::DevicesInfo log support @@ -966,8 +983,8 @@ * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs */ "dhcp_host_name": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } } diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template index e79d4820ad..abe295283a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template @@ -2,54 +2,50 @@ "template": "error_index*", "mappings": { "error_doc": { - "_timestamp": { - "enabled": true - }, "properties": { "exception": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "hostname": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "stack": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "timestamp": { "type": "date", "format": "epoch_millis" }, "message": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "raw_message": { - "type": "string", - "index": "not_analyzed", - "ignore_above": 8191 + "type": "text", + "index": "false" }, "raw_message_bytes": { "type": "binary", "index": "no" }, "error_fields": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "error_hash": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "failed_sensor_type": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "error_type": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "alert": { "type": "nested" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping index c42343ef0f..a3451ce8e5 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping @@ -1,33 +1,30 @@ { "mappings": { "metaalert_doc": { - "_timestamp": { - "enabled": true - }, "dynamic_templates": [ { "alert_template": { "path_match": "alert.*", "match_mapping_type": "string", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } } ], "properties": { "guid": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "score": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "status": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "timestamp": { "type": "date", diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template index f13a9ee13b..7ac4d16df6 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template @@ -2,9 +2,6 @@ "template": "snort_index*", "mappings": { "snort_doc": { - "_timestamp": { - "enabled": true - }, "dynamic_templates": [ { "geo_location_point": { @@ -20,8 +17,8 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -30,8 +27,8 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -40,8 +37,8 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -50,8 +47,8 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -60,8 +57,8 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" } } }, @@ -105,7 +102,8 @@ { "threat_triage_reason": { "mapping": { - "type": "string" + "type": "text", + "fielddata": "true" }, "match": "threat.triage.rules:*:reason", "match_mapping_type": "*" @@ -114,7 +112,8 @@ { "threat_triage_name": { "mapping": { - "type": "string" + "type": "text", + "fielddata": "true" }, "match": "threat.triage.rules:*:name", "match_mapping_type": "*" @@ -127,8 +126,8 @@ "format": "epoch_millis" }, "source:type": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "ip_dst_addr": { "type": "ip" @@ -146,16 +145,16 @@ "type": "integer" }, "ethdst": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "ethlen": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "ethsrc": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "id": { "type": "integer" @@ -167,33 +166,39 @@ "type": "boolean" }, "msg": { - "type": "string" + "type": "text", + "fielddata": "true" }, "protocol": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "sig_generator": { - "type": "string", - "index": "not_analyzed" + "type": "text", + "index": "false" }, "sig_id": { "type": "integer" }, "sig_rev": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tcpack": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tcpflags": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tcpseq": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tcpwindow": { - "type": "string" + "type": "text", + "fielddata": "true" }, "threat:triage:level": { "type": "double" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template index e9a2cd0227..00a5eeec6e 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template @@ -102,7 +102,8 @@ { "threat_triage_reason": { "mapping": { - "type": "text" + "type": "text", + "fielddata": "true" }, "match": "threat.triage.rules:*:reason", "match_mapping_type": "*" @@ -111,7 +112,8 @@ { "threat_triage_name": { "mapping": { - "type": "text" + "type": "text", + "fielddata": "true" }, "match": "threat.triage.rules:*:name", "match_mapping_type": "*" @@ -182,7 +184,8 @@ "index": "false" }, "riflags": { - "type": "text" + "type": "text", + "fielddata": "true" }, "ruflags": { "type": "text", @@ -221,7 +224,8 @@ "index": "false" }, "end-reason": { - "type": "text" + "type": "text", + "fielddata": "true" }, "alert": { "type": "nested" From f3475bf7a50c733241f2aaf31dc03c08efcc89ad Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 23 Oct 2017 10:58:26 -0400 Subject: [PATCH 18/59] updating tests. --- metron-platform/metron-indexing/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml index 55fe45578b..40fcd69f7d 100644 --- a/metron-platform/metron-indexing/pom.xml +++ b/metron-platform/metron-indexing/pom.xml @@ -31,6 +31,12 @@ ${global_hbase_guava_version} + + org.apache.logging.log4j + log4j-core + 2.8.2 + test + org.apache.metron metron-common @@ -167,6 +173,7 @@ test-jar test + From 12354430487f2a02fd50bc5ac2a6aa24c141dd6b Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 23 Oct 2017 12:21:01 -0400 Subject: [PATCH 19/59] Removing log4j-core. --- metron-analytics/metron-profiler-client/pom.xml | 4 ++++ metron-analytics/metron-profiler-common/pom.xml | 4 ++++ metron-analytics/metron-profiler/pom.xml | 4 ++++ metron-platform/metron-common/pom.xml | 4 ++++ metron-platform/metron-elasticsearch/pom.xml | 4 ++-- metron-platform/metron-enrichment/pom.xml | 4 ++++ metron-platform/metron-hbase/pom.xml | 4 ++++ metron-platform/metron-indexing/pom.xml | 4 ++++ metron-platform/metron-integration-test/pom.xml | 4 ++++ metron-platform/metron-parsers/pom.xml | 4 ++++ metron-platform/metron-pcap-backend/pom.xml | 4 ++++ metron-platform/metron-pcap/pom.xml | 4 ++++ metron-platform/metron-storm-kafka-override/pom.xml | 4 ++++ metron-platform/metron-storm-kafka/pom.xml | 4 ++++ metron-platform/metron-test-utilities/pom.xml | 4 ++++ metron-platform/metron-writer/pom.xml | 4 ++++ 16 files changed, 62 insertions(+), 2 deletions(-) diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml index 69b8c298f8..c9c8940ce2 100644 --- a/metron-analytics/metron-profiler-client/pom.xml +++ b/metron-analytics/metron-profiler-client/pom.xml @@ -139,6 +139,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-analytics/metron-profiler-common/pom.xml b/metron-analytics/metron-profiler-common/pom.xml index 58ec3fb6dc..0ca5ce3bfc 100644 --- a/metron-analytics/metron-profiler-common/pom.xml +++ b/metron-analytics/metron-profiler-common/pom.xml @@ -114,6 +114,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + log4j-slf4j-impl org.apache.logging.log4j diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index e1ee806da6..816cd8b729 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -202,6 +202,10 @@ storm-core ${global_storm_version} + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index 305488124e..2c1055271f 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -84,6 +84,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 5de1d863b4..e52c1d9e8a 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -227,11 +227,11 @@ log4j-api 2.8.2 - + diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index dd3998b6c0..993b5e1071 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -199,6 +199,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-hbase/pom.xml b/metron-platform/metron-hbase/pom.xml index 180134f995..38cf492582 100644 --- a/metron-platform/metron-hbase/pom.xml +++ b/metron-platform/metron-hbase/pom.xml @@ -139,6 +139,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml index 40fcd69f7d..2fb45724cd 100644 --- a/metron-platform/metron-indexing/pom.xml +++ b/metron-platform/metron-indexing/pom.xml @@ -99,6 +99,10 @@ log4j-slf4j-impl org.apache.logging.log4j + + org.apache.logging.log4j + log4j-core + diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 0d99426975..1d8a9967b5 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -53,6 +53,10 @@ log4j-slf4j-impl org.apache.logging.log4j + + org.apache.logging.log4j + log4j-core + diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index 85c621862d..b4998e84b4 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -129,6 +129,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index 5878873bc3..fd0156e8dd 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -173,6 +173,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml index 39e8588739..9170a3162c 100644 --- a/metron-platform/metron-pcap/pom.xml +++ b/metron-platform/metron-pcap/pom.xml @@ -91,6 +91,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-storm-kafka-override/pom.xml b/metron-platform/metron-storm-kafka-override/pom.xml index be1545961b..1087982ce3 100644 --- a/metron-platform/metron-storm-kafka-override/pom.xml +++ b/metron-platform/metron-storm-kafka-override/pom.xml @@ -46,6 +46,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-storm-kafka/pom.xml b/metron-platform/metron-storm-kafka/pom.xml index 435c4a299a..99619e5d94 100644 --- a/metron-platform/metron-storm-kafka/pom.xml +++ b/metron-platform/metron-storm-kafka/pom.xml @@ -51,6 +51,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-test-utilities/pom.xml b/metron-platform/metron-test-utilities/pom.xml index 250276032a..b32ef54f75 100644 --- a/metron-platform/metron-test-utilities/pom.xml +++ b/metron-platform/metron-test-utilities/pom.xml @@ -106,6 +106,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml index de6b3b8118..a91dcbc766 100644 --- a/metron-platform/metron-writer/pom.xml +++ b/metron-platform/metron-writer/pom.xml @@ -129,6 +129,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + servlet-api javax.servlet From 46e0a05b34e43fb2a42aa3ecb5f509678534ae79 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 23 Oct 2017 17:48:31 -0400 Subject: [PATCH 20/59] Updating poms to exclude the storm log4j. --- .../metron-profiler-client/pom.xml | 4 ++++ .../metron-profiler-common/pom.xml | 4 ++++ metron-analytics/metron-profiler/pom.xml | 4 ++++ metron-platform/elasticsearch-shaded/pom.xml | 5 +++++ .../META-INF/log4j-provider.properties | 18 ++++++++++++++++++ metron-platform/metron-common/pom.xml | 4 ++++ metron-platform/metron-elasticsearch/pom.xml | 6 +++--- metron-platform/metron-enrichment/pom.xml | 4 ++++ metron-platform/metron-hbase/pom.xml | 4 ++++ .../metron-integration-test/pom.xml | 4 ++++ metron-platform/metron-parsers/pom.xml | 4 ++++ metron-platform/metron-pcap-backend/pom.xml | 4 ++++ metron-platform/metron-pcap/pom.xml | 4 ++++ metron-platform/metron-solr/pom.xml | 8 ++++++++ .../metron-storm-kafka-override/pom.xml | 4 ++++ metron-platform/metron-storm-kafka/pom.xml | 4 ++++ metron-platform/metron-test-utilities/pom.xml | 4 ++++ metron-platform/metron-writer/pom.xml | 4 ++++ 18 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 metron-platform/elasticsearch-shaded/src/main/resources/META-INF/log4j-provider.properties diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml index c9c8940ce2..e8732606cb 100644 --- a/metron-analytics/metron-profiler-client/pom.xml +++ b/metron-analytics/metron-profiler-client/pom.xml @@ -139,6 +139,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-analytics/metron-profiler-common/pom.xml b/metron-analytics/metron-profiler-common/pom.xml index 0ca5ce3bfc..a11ee7bfb8 100644 --- a/metron-analytics/metron-profiler-common/pom.xml +++ b/metron-analytics/metron-profiler-common/pom.xml @@ -114,6 +114,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index 816cd8b729..8d6d62d4ed 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -202,6 +202,10 @@ storm-core ${global_storm_version} + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index ac5c2bb9a0..ed97261ee0 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -92,6 +92,11 @@ log4j-core 2.8.2 + + org.apache.logging.log4j + log4j-api + 2.8.2 + org.elasticsearch elasticsearch diff --git a/metron-platform/elasticsearch-shaded/src/main/resources/META-INF/log4j-provider.properties b/metron-platform/elasticsearch-shaded/src/main/resources/META-INF/log4j-provider.properties new file mode 100644 index 0000000000..c4bd3f06b6 --- /dev/null +++ b/metron-platform/elasticsearch-shaded/src/main/resources/META-INF/log4j-provider.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LoggerContextFactory = org.apache.metron.logging.log4j.core.impl.Log4jContextFactory +Log4jAPIVersion = 2.6.0 +FactoryPriority= 10 \ No newline at end of file diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index 2c1055271f..9a00cdaab9 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -88,6 +88,10 @@ org.apache.logging.log4j log4j-core + + org.apache.logging.log4j + log4j-api + servlet-api javax.servlet diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index e52c1d9e8a..127cf33e6f 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -225,13 +225,13 @@ org.apache.logging.log4j log4j-api - 2.8.2 + ${global_log4j_core_version} - + diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index 993b5e1071..b65f15df40 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -199,6 +199,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-platform/metron-hbase/pom.xml b/metron-platform/metron-hbase/pom.xml index 38cf492582..b7ec006248 100644 --- a/metron-platform/metron-hbase/pom.xml +++ b/metron-platform/metron-hbase/pom.xml @@ -139,6 +139,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 1d8a9967b5..8d3cdd8073 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -57,6 +57,10 @@ org.apache.logging.log4j log4j-core + + org.apache.logging.log4j + log4j-api + diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index b4998e84b4..909fc55ecd 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -129,6 +129,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index fd0156e8dd..e30a957127 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -173,6 +173,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml index 9170a3162c..ea5d445e21 100644 --- a/metron-platform/metron-pcap/pom.xml +++ b/metron-platform/metron-pcap/pom.xml @@ -91,6 +91,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml index 97132c4117..487faa102d 100644 --- a/metron-platform/metron-solr/pom.xml +++ b/metron-platform/metron-solr/pom.xml @@ -71,6 +71,14 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-api + servlet-api javax.servlet diff --git a/metron-platform/metron-storm-kafka-override/pom.xml b/metron-platform/metron-storm-kafka-override/pom.xml index 1087982ce3..3ee7244592 100644 --- a/metron-platform/metron-storm-kafka-override/pom.xml +++ b/metron-platform/metron-storm-kafka-override/pom.xml @@ -50,6 +50,10 @@ org.apache.logging.log4j log4j-core + + org.apache.logging.log4j + log4j-api + servlet-api javax.servlet diff --git a/metron-platform/metron-storm-kafka/pom.xml b/metron-platform/metron-storm-kafka/pom.xml index 99619e5d94..5821682ab3 100644 --- a/metron-platform/metron-storm-kafka/pom.xml +++ b/metron-platform/metron-storm-kafka/pom.xml @@ -51,6 +51,10 @@ ${global_storm_version} provided + + org.apache.logging.log4j + log4j-api + org.apache.logging.log4j log4j-core diff --git a/metron-platform/metron-test-utilities/pom.xml b/metron-platform/metron-test-utilities/pom.xml index b32ef54f75..97eb10934a 100644 --- a/metron-platform/metron-test-utilities/pom.xml +++ b/metron-platform/metron-test-utilities/pom.xml @@ -110,6 +110,10 @@ org.apache.logging.log4j log4j-core + + org.apache.logging.log4j + log4j-api + servlet-api javax.servlet diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml index a91dcbc766..6ccd943f16 100644 --- a/metron-platform/metron-writer/pom.xml +++ b/metron-platform/metron-writer/pom.xml @@ -133,6 +133,10 @@ org.apache.logging.log4j log4j-core + + org.apache.logging.log4j + log4j-api + servlet-api javax.servlet From 798f954ed6fb8598a27f37173a3f0ef66f0164b7 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 23 Oct 2017 19:47:43 -0400 Subject: [PATCH 21/59] Updating pom to include log4j --- metron-analytics/metron-profiler/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index 8d6d62d4ed..4eed28bb5c 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -229,6 +229,16 @@ provided + + org.apache.logging.log4j + log4j-api + ${global_log4j_core_version} + + + org.apache.logging.log4j + log4j-core + ${global_log4j_core_version} + org.apache.kafka kafka_2.10 From 9e2de3bf883b710a1c50f14f2fd4e119ff88e24a Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 23 Oct 2017 20:28:40 -0400 Subject: [PATCH 22/59] Adding back in teh log4j but with the right version. --- metron-analytics/metron-profiler/pom.xml | 2 ++ metron-platform/metron-enrichment/pom.xml | 13 ++++++++++++- metron-platform/metron-parsers/pom.xml | 12 ++++++++++++ metron-platform/metron-pcap-backend/pom.xml | 12 ++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index 4eed28bb5c..d06617b2a1 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -233,11 +233,13 @@ org.apache.logging.log4j log4j-api ${global_log4j_core_version} + test org.apache.logging.log4j log4j-core ${global_log4j_core_version} + test org.apache.kafka diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index b65f15df40..feefb63e5b 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -221,7 +221,18 @@ - + + org.apache.logging.log4j + log4j-api + ${global_log4j_core_version} + test + + + org.apache.logging.log4j + log4j-core + ${global_log4j_core_version} + test + com.google.guava guava diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index 909fc55ecd..3fe237ce74 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -151,6 +151,18 @@ + + org.apache.logging.log4j + log4j-api + ${global_log4j_core_version} + test + + + org.apache.logging.log4j + log4j-core + ${global_log4j_core_version} + test + junit junit diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index e30a957127..dc199e898c 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -195,6 +195,18 @@ + + org.apache.logging.log4j + log4j-api + ${global_log4j_core_version} + test + + + org.apache.logging.log4j + log4j-core + ${global_log4j_core_version} + test + org.apache.metron metron-pcap From c160837ae2863e71c5d392499af575ac1893fb0c Mon Sep 17 00:00:00 2001 From: cstella Date: Tue, 24 Oct 2017 09:42:55 -0400 Subject: [PATCH 23/59] Updating dependencies with url. --- dependencies_with_url.csv | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index b95f25b44c..96b0aaa19b 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -309,3 +309,26 @@ org.springframework.security.kerberos:spring-security-kerberos-core:jar:1.0.1.RE org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j +com.github.spullara.mustache.java:compiler:jar:0.9.3:compile,ASLv2,https://github.com/spullara/mustache.java/blob/master/LICENSE +io.netty:netty-buffer:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-codec-http:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-codec:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-common:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-handler:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty:jar:3.10.6.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-resolver:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-transport:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +joda-time:joda-time:jar:2.9.5:compile,ASLv2,https://github.com/JodaOrg/joda-time +net.sf.jopt-simple:jopt-simple:jar:5.0.2:compile,The MIT License,http://jopt-simple.sourceforge.net +org.elasticsearch.client:elasticsearch-rest-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.client:transport:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch:elasticsearch:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch:jna:jar:4.4.0-1:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:lang-mustache-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:parent-join-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:percolator-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:reindex-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:transport-netty3-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:transport-netty4-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch:securesm:jar:1.1:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.hdrhistogram:HdrHistogram:jar:2.1.9:compile,BSD,https://github.com/HdrHistogram/HdrHistogram/blob/master/LICENSE.txt From a134e230454db7d4dfff0ed8a308ab205efa4285 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 24 Oct 2017 07:58:37 -0600 Subject: [PATCH 24/59] get netty conflicts with rest api working --- metron-platform/elasticsearch-shaded/pom.xml | 27 +++----------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index ac5c2bb9a0..e7529014e1 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -29,29 +29,6 @@ guava 18.0 - org.elasticsearch.client transport @@ -167,6 +144,10 @@ + + io.netty + org.apache.metron.io.netty + org.apache.logging.log4j org.apache.metron.logging.log4j From e13688fbfcb09624fcad439959c285dd41e32a0b Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 24 Oct 2017 11:12:54 -0600 Subject: [PATCH 25/59] Fix text/keyword types according to ES mapping recommendations --- .../CURRENT/package/files/bro_index.template | 325 ++++++------------ .../package/files/error_index.template | 31 +- ...meta_index.mapping => meta_index.template} | 13 +- .../package/files/snort_index.template | 33 +- .../CURRENT/package/files/yaf_index.template | 57 +-- .../package/scripts/indexing_master.py | 6 +- .../package/scripts/params/params_linux.py | 2 +- 7 files changed, 160 insertions(+), 307 deletions(-) rename metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/{meta_index.mapping => meta_index.template} (71%) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template index fa7bc9a578..84ab1703bf 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template @@ -17,8 +17,7 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -27,8 +26,7 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -37,8 +35,7 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -47,8 +44,7 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -57,8 +53,7 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -136,8 +131,7 @@ * Metron-specific fields */ "source:type": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * Widely-used Bro fields (potentially renamed during Metron ingest) @@ -147,8 +141,7 @@ "format": "epoch_millis" }, "uid": { - "type": "text", - "index": "false" + "type": "keyword" }, "alert": { "type": "nested" @@ -195,24 +188,20 @@ "type": "integer" }, "method": { - "type": "text", - "index": "false" + "type": "keyword" }, "host": { - "type": "text", - "index": "false" + "type": "keyword" }, "uri": { - "type": "text", - "index": "false" + "type": "keyword", + "ignore_above": 8191 }, "referrer": { - "type": "text", - "index": "false" + "type": "keyword" }, "version": { - "type": "text", - "index": "false" + "type": "keyword" }, "user_agent": { "type": "text", @@ -228,16 +217,13 @@ "type": "integer" }, "status_msg": { - "type": "text", - "index": "false" + "type": "keyword" }, "username": { - "type": "text", - "index": "false" + "type": "keyword" }, "password": { - "type": "text", - "index": "false" + "type": "keyword" }, "capture_password": { "type": "boolean" @@ -254,36 +240,31 @@ * Notes: Field exists in the DNS and DHCP logs */ "proto": { - "type": "text", - "index": "false" + "type": "keyword" }, "trans_id": { "type": "long" }, "query": { - "type": "text", - "index": "false" + "type": "keyword" }, "qclass": { "type": "integer" }, "qclass_name": { - "type": "text", - "index": "false" + "type": "keyword" }, "qtype": { "type": "integer" }, "qtype_name": { - "type": "text", - "index": "false" + "type": "keyword" }, "rcode": { "type": "integer" }, "rcode_name": { - "type": "text", - "index": "false" + "type": "keyword" }, "AA": { "type": "boolean" @@ -322,58 +303,46 @@ * Notes: Field exists in the Conn and Files logs */ "service": { - "type": "text", - "index": "false" + "type": "keyword" }, "duration": { "type": "float" }, "orig_bytes": { - "type": "long", - "index": "false" + "type": "long" }, "resp_bytes": { - "type": "long", - "index": "false" + "type": "long" }, "conn_state": { - "type": "text", - "index": "false" + "type": "keyword" }, "local_orig": { "type": "boolean" }, "local_resp": { - "type": "text", - "index": "false" + "type": "keyword" }, "missed_bytes": { - "type": "long", - "index": "false" + "type": "long" }, "history": { - "type": "text", - "index": "false" + "type": "keyword" }, "orig_pkts": { - "type": "long", - "index": "false" + "type": "long" }, "orig_ip_bytes": { - "type": "long", - "index": "false" + "type": "long" }, "resp_pkts": { - "type": "long", - "index": "false" + "type": "long" }, "resp_ip_bytes": { - "type": "long", - "index": "false" + "type": "long" }, "tunnel_parents": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * DPD log support @@ -384,12 +353,10 @@ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs */ "analyzer": { - "type": "text", - "index": "false" + "type": "keyword" }, "failure_reason": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * FTP log support @@ -409,12 +376,10 @@ * Notes: Field exists in the FTP and Notice logs */ "user": { - "type": "text", - "index": "false" + "type": "keyword" }, "command": { - "type": "text", - "index": "false" + "type": "keyword" }, "arg": { "type": "text", @@ -433,8 +398,7 @@ "type": "integer" }, "reply_msg": { - "type": "text", - "index": "false" + "type": "keyword" }, "data_channel:passive": { "type": "boolean" @@ -457,8 +421,7 @@ "type": "boolean" }, "fuid": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * Files log support @@ -480,8 +443,7 @@ "analyzer": "simple" }, "source": { - "type": "text", - "index": "false" + "type": "keyword" }, "depth": { "type": "integer" @@ -492,8 +454,7 @@ "analyzer": "simple" }, "filename": { - "type": "text", - "index": "false" + "type": "keyword" }, "is_orig": { "type": "boolean" @@ -514,20 +475,16 @@ "type": "boolean" }, "parent_fuid": { - "type": "text", - "index": "false" + "type": "keyword" }, "md5": { - "type": "text", - "index": "false" + "type": "keyword" }, "sha1": { - "type": "text", - "index": "false" + "type": "keyword" }, "sha256": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * Known::CertInfo log support @@ -551,8 +508,7 @@ "analyzer": "simple" }, "serial": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * SMTP log support @@ -578,8 +534,7 @@ "analyzer": "simple" }, "date": { - "type": "text", - "index": "false" + "type": "keyword" }, "from": { "type": "text", @@ -597,12 +552,10 @@ "analyzer": "simple" }, "msg_id": { - "type": "text", - "index": "false" + "type": "keyword" }, "in_reply_to": { - "type": "text", - "index": "false" + "type": "keyword" }, "x_originating_ip": { "type": "ip" @@ -623,15 +576,13 @@ "analyzer": "simple" }, "path": { - "type": "text", - "index": "false" + "type": "keyword" }, "tls": { "type": "boolean" }, "fuids": { - "type": "text", - "index": "false" + "type": "keyword" }, "is_webmail": { "type": "boolean" @@ -645,27 +596,22 @@ * Notes: Field exists in the HTTP, SSL, and SSH logs */ "cipher": { - "type": "text", - "index": "false" + "type": "keyword" }, "curve": { - "type": "text", - "index": "false" + "type": "keyword" }, "server_name": { - "type": "text", - "index": "false" + "type": "keyword" }, "resumed": { "type": "boolean" }, "last_alert": { - "type": "text", - "index": "false" + "type": "keyword" }, "next_protocol": { - "type": "text", - "index": "false" + "type": "keyword" }, "established": { "type": "boolean" @@ -675,19 +621,16 @@ * https://www.bro.org/sphinx/scripts/base/frameworks/notice/weird.bro.html#type-Weird::Info */ "name": { - "type": "text", - "index": "false" + "type": "keyword" }, "addl": { - "type": "text", - "index": "false" + "type": "keyword" }, "notice": { "type": "boolean" }, "peer": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * Notice log support @@ -701,24 +644,19 @@ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs */ "file_mime_type": { - "type": "text", - "index": "false" + "type": "keyword" }, "file_desc": { - "type": "text", - "index": "false" + "type": "keyword" }, "note": { - "type": "text", - "index": "false" + "type": "keyword" }, "msg": { - "type": "text", - "index": "false" + "type": "keyword" }, "sub": { - "type": "text", - "index": "false" + "type": "keyword" }, "src": { "type": "ip" @@ -727,27 +665,22 @@ "type": "ip" }, "p": { - "type": "integer", - "index": "false" + "type": "integer" }, "n": { - "type": "integer", - "index": "false" + "type": "integer" }, "src_peer": { "type": "ip" }, "peer_descr": { - "type": "text", - "index": "false" + "type": "keyword" }, "actions": { - "type": "text", - "index": "false" + "type": "keyword" }, "suppress_for": { - "type": "double", - "index": "false" + "type": "double" }, "dropped": { "type": "boolean" @@ -764,15 +697,13 @@ * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs */ "mac": { - "type": "text", - "index": "false" + "type": "keyword" }, "assigned_ip": { "type": "ip" }, "lease_time": { - "type": "float", - "index": "false" + "type": "float" }, /* * SSH log support @@ -786,44 +717,34 @@ "type": "boolean" }, "auth_attempts": { - "type": "integer", - "index": "false" + "type": "integer" }, "direction": { - "type": "text", - "index": "false" + "type": "keyword" }, "client": { - "type": "text", - "index": "false" + "type": "keyword" }, "server": { - "type": "text", - "index": "false" + "type": "keyword" }, "cipher_alg": { - "type": "text", - "index": "false" + "type": "keyword" }, "mac_alg": { - "type": "text", - "index": "false" + "type": "keyword" }, "compression_alg": { - "type": "text", - "index": "false" + "type": "keyword" }, "kex_alg": { - "type": "text", - "index": "false" + "type": "keyword" }, "host_key_alg": { - "type": "text", - "index": "false" + "type": "keyword" }, "host_key": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * Software log support @@ -834,32 +755,25 @@ * Notes: Field exists in the HTTP and Software logs */ "host_p": { - "type": "integer", - "index": "false" + "type": "integer" }, "software_type": { - "type": "text", - "index": "false" + "type": "keyword" }, "version:major": { - "type": "text", - "index": "false" + "type": "keyword" }, "version:minor": { - "type": "text", - "index": "false" + "type": "keyword" }, "version:minor2": { - "type": "text", - "index": "false" + "type": "keyword" }, "version:minor3": { - "type": "text", - "index": "false" + "type": "keyword" }, "version:addl": { - "type": "text", - "index": "false" + "type": "keyword" }, "unparsed_version": { "type": "text", @@ -881,12 +795,10 @@ "type": "ip" }, "connect_info": { - "type": "text", - "index": "false" + "type": "keyword" }, "result": { - "type": "text", - "index": "false" + "type": "keyword" }, /* * X509 log support @@ -900,79 +812,61 @@ * logs, however, id is a string to identify the certificate file id. */ "id": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:version": { - "type": "integer", - "index": "false" + "type": "integer" }, "certificate:serial": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:subject": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:issuer": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:not_valid_before": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:not_valid_after": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:key_alg": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:sig_alg": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:key_type": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:key_length": { - "type": "integer", - "index": "false" + "type": "integer" }, "certificate:exponent": { - "type": "text", - "index": "false" + "type": "keyword" }, "certificate:curve": { - "type": "text", - "index": "false" + "type": "keyword" }, "san:dns": { - "type": "text", - "index": "false" + "type": "keyword" }, "san:uri": { - "type": "text", - "index": "false" + "type": "keyword" }, "san:email": { - "type": "text", - "index": "false" + "type": "keyword" }, "san:ip": { - "type": "text", - "index": "false" + "type": "keyword" }, "basic_constraints:ca": { "type": "boolean" }, "basic_constraints:path_len": { - "type": "integer", - "index": "false" + "type": "integer" }, /* * Known::DevicesInfo log support @@ -983,8 +877,7 @@ * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs */ "dhcp_host_name": { - "type": "text", - "index": "false" + "type": "keyword" } } } diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template index abe295283a..00aaf87dd3 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template @@ -4,48 +4,39 @@ "error_doc": { "properties": { "exception": { - "type": "text", - "index": "false" + "type": "keyword" }, "hostname": { - "type": "text", - "index": "false" + "type": "keyword" }, "stack": { - "type": "text", - "index": "false" + "type": "keyword" }, "timestamp": { "type": "date", "format": "epoch_millis" }, "message": { - "type": "text", - "index": "false" + "type": "keyword" }, "raw_message": { - "type": "text", - "index": "false" + "type": "keyword", + "ignore_above": 8191 }, "raw_message_bytes": { - "type": "binary", - "index": "no" + "type": "binary" }, "error_fields": { - "type": "text", - "index": "false" + "type": "keyword" }, "error_hash": { - "type": "text", - "index": "false" + "type": "keyword" }, "failed_sensor_type": { - "type": "text", - "index": "false" + "type": "keyword" }, "error_type": { - "type": "text", - "index": "false" + "type": "keyword" }, "alert": { "type": "nested" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template similarity index 71% rename from metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping rename to metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template index a3451ce8e5..d26a2e78b5 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template @@ -1,4 +1,5 @@ { + "template": "metaalert_index*", "mappings": { "metaalert_doc": { "dynamic_templates": [ @@ -7,24 +8,20 @@ "path_match": "alert.*", "match_mapping_type": "string", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } } ], "properties": { "guid": { - "type": "text", - "index": "false" + "type": "keyword" }, "score": { - "type": "text", - "index": "false" + "type": "keyword" }, "status": { - "type": "text", - "index": "false" + "type": "keyword" }, "timestamp": { "type": "date", diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template index 7ac4d16df6..c18a22eb08 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template @@ -17,8 +17,7 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -27,8 +26,7 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -37,8 +35,7 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -47,8 +44,7 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -57,8 +53,7 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -126,8 +121,7 @@ "format": "epoch_millis" }, "source:type": { - "type": "text", - "index": "false" + "type": "keyword" }, "ip_dst_addr": { "type": "ip" @@ -145,16 +139,13 @@ "type": "integer" }, "ethdst": { - "type": "text", - "index": "false" + "type": "keyword" }, "ethlen": { - "type": "text", - "index": "false" + "type": "keyword" }, "ethsrc": { - "type": "text", - "index": "false" + "type": "keyword" }, "id": { "type": "integer" @@ -170,12 +161,10 @@ "fielddata": "true" }, "protocol": { - "type": "text", - "index": "false" + "type": "keyword" }, "sig_generator": { - "type": "text", - "index": "false" + "type": "keyword" }, "sig_id": { "type": "integer" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template index 00a5eeec6e..d2e7534758 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template @@ -17,8 +17,7 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -27,8 +26,7 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -37,8 +35,7 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -47,8 +44,7 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -57,8 +53,7 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "text", - "index": "false" + "type": "keyword" } } }, @@ -126,8 +121,7 @@ "format": "epoch_millis" }, "source:type": { - "type": "text", - "index": "false" + "type": "keyword" }, "ip_dst_addr": { "type": "ip" @@ -156,56 +150,44 @@ "type": "double" }, "proto": { - "type": "text", - "index": "false" + "type": "keyword" }, "sip": { - "type": "text", - "index": "false" + "type": "keyword" }, "sp": { - "type": "text", - "index": "false" + "type": "keyword" }, "dip": { - "type": "text", - "index": "false" + "type": "keyword" }, "dp": { - "type": "text", - "index": "false" + "type": "keyword" }, "iflags": { - "type": "text", - "index": "false" + "type": "keyword" }, "uflags": { - "type": "text", - "index": "false" + "type": "keyword" }, "riflags": { "type": "text", "fielddata": "true" }, "ruflags": { - "type": "text", - "index": "false" + "type": "keyword" }, "isn": { - "type": "text", - "index": "false" + "type": "keyword" }, "risn": { - "type": "text", - "index": "false" + "type": "keyword" }, "tag": { - "type": "text", - "index": "false" + "type": "keyword" }, "rtag": { - "type": "text", - "index": "false" + "type": "keyword" }, "pkt": { "type": "integer" @@ -220,8 +202,7 @@ "type": "integer" }, "app": { - "type": "text", - "index": "false" + "type": "keyword" }, "end-reason": { "type": "text", diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 371cab0285..6436da2443 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -129,7 +129,7 @@ def elasticsearch_template_install(self, env): File(params.meta_index_path, mode=0755, - content=StaticFile('meta_index.mapping') + content=StaticFile('meta_index.template') ) bro_cmd = ambari_format( @@ -145,7 +145,7 @@ def elasticsearch_template_install(self, env): 'curl -s -XPOST http://{es_http_url}/_template/error_index -d @{error_index_path}') Execute(error_cmd, logoutput=True) error_cmd = ambari_format( - 'curl -s -XPOST http://{es_http_url}/metaalerts -d @{meta_index_path}') + 'curl -s -XPOST http://{es_http_url}/_template/metaalerts -d @{meta_index_path}') Execute(error_cmd, logoutput=True) def elasticsearch_template_delete(self, env): @@ -160,6 +160,8 @@ def elasticsearch_template_delete(self, env): Execute(yaf_cmd, logoutput=True) error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/error_index*"') Execute(error_cmd, logoutput=True) + error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/metaalerts*"') + Execute(error_cmd, logoutput=True) def zeppelin_notebook_import(self, env): from params import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index b73c3ef6fb..6011281a64 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -193,7 +193,7 @@ snort_index_path = tmp_dir + "/snort_index.template" yaf_index_path = tmp_dir + "/yaf_index.template" error_index_path = tmp_dir + "/error_index.template" -meta_index_path = tmp_dir + "/meta_index.mapping" +meta_index_path = tmp_dir + "/meta_index.template" # Zeppelin Notebooks metron_config_zeppelin_path = format("{metron_config_path}/zeppelin") From 5ad3e6add58f2ded55aa3ee7eee2a81819282610 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Tue, 24 Oct 2017 14:12:21 -0400 Subject: [PATCH 26/59] Moved PAM limit configuration from Ansible to the MPack --- .../5.6.2/configuration/elastic-env.xml | 21 ++++++++- .../5.6.2/package/scripts/elastic.py | 28 +++++++++--- .../5.6.2/package/scripts/elastic_master.py | 13 +++--- .../5.6.2/package/scripts/elastic_slave.py | 11 ++--- .../5.6.2/package/scripts/params.py | 7 +++ .../5.6.2/package/scripts/service_check.py | 15 +++---- .../5.6.2/package/scripts/slave.py | 26 ++++++++--- .../templates/elasticsearch_limits.conf.j2 | 20 +++++++++ .../roles/ambari_common/meta/main.yml | 1 - metron-deployment/roles/limits/tasks/main.yml | 45 ------------------- 10 files changed, 108 insertions(+), 79 deletions(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 delete mode 100644 metron-deployment/roles/limits/tasks/main.yml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml index a4de039abd..9e4f8ad266 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml @@ -49,7 +49,6 @@ /var/run/elasticsearch The directory for pid files - content @@ -64,4 +63,24 @@ export JAVA_HOME={{java64_home}} export PATH=$PATH:$JAVA_HOME/bin + + elastic_user_nofile_limit + 65536 + Max open file limit for Elasticsearch user. + + + elastic_user_nproc_limit + 2048 + Max number of processes for Elasticsearch user. + + + elastic_user_memlock_soft_limit + unlimited + Max locked-in memory address space (soft memlock limit). + + + elastic_user_memlock_hard_limit + unlimited + Max locked-in memory address space (hard memlock limit). + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py index c44d5ef686..e27e8bf7a1 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py @@ -23,18 +23,22 @@ from resource_management.core.source import InlineTemplate from resource_management.core.source import Template from resource_management.core.resources import User +from resource_management.core.logger import Logger +from resource_management.libraries.functions import format as ambari_format def elastic(): import params + Logger.info("Creating user: {0}:{1}".format(params.elastic_user, params.elastic_group)) User(params.elastic_user, action = "create", groups = params.elastic_group) + params.path_data = params.path_data.replace('"', '') data_path = params.path_data.replace(' ', '').split(',') data_path[:] = [x.replace('"', '') for x in data_path] - directories = [params.log_dir, params.pid_dir, params.conf_dir] directories = directories + data_path + ["{0}/scripts".format(params.conf_dir)] + Logger.info("Creating directories: {0}".format(directories)) Directory(directories, create_parents=True, mode=0755, @@ -42,7 +46,7 @@ def elastic(): group=params.elastic_group ) - print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir) + Logger.info("Master env: ""{0}/elastic-env.sh".format(params.conf_dir)) File("{0}/elastic-env.sh".format(params.conf_dir), owner=params.elastic_user, group=params.elastic_group, @@ -50,8 +54,7 @@ def elastic(): ) configurations = params.config['configurations']['elastic-site'] - - print "Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir) + Logger.info("Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir)) File("{0}/elasticsearch.yml".format(params.conf_dir), content=Template( "elasticsearch.master.yaml.j2", @@ -60,9 +63,24 @@ def elastic(): group=params.elastic_group ) - print "Master sysconfig: /etc/sysconfig/elasticsearch" + Logger.info("Master sysconfig: /etc/sysconfig/elasticsearch") File("/etc/sysconfig/elasticsearch", owner="root", group="root", content=InlineTemplate(params.sysconfig_template) ) + + # in some OS this folder may not exist, so create it + Logger.info("Ensure PAM limits directory exists: {0}".format(params.limits_conf_dir)) + Directory(params.limits_conf_dir, + create_parents=True, + owner='root', + group='root' + ) + + Logger.info("Master PAM limits: {0}".format(params.limits_conf_file)) + File(params.limits_conf_file, + content=Template('elasticsearch_limits.conf.j2'), + owner="root", + group="root" + ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py index 3cd63c302e..c3f089afd4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py @@ -28,44 +28,43 @@ class Elasticsearch(Script): def install(self, env): import params env.set_params(params) - Logger.info('Install ES Master Node') + Logger.info('Install Elasticsearch master node') self.install_packages(env) def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) - + Logger.info('Configure Elasticsearch master node') elastic() def stop(self, env, upgrade_type=None): import params env.set_params(params) + Logger.info('Stop Elasticsearch master node') stop_cmd = "service elasticsearch stop" - print 'Stop the Master' Execute(stop_cmd) def start(self, env, upgrade_type=None): import params env.set_params(params) - + Logger.info('Start Elasticsearch master node') self.configure(env) start_cmd = "service elasticsearch start" - print 'Start the Master' Execute(start_cmd) def status(self, env): import params env.set_params(params) + Logger.info('Check status of Elasticsearch master node') status_cmd = "service elasticsearch status" - print 'Status of the Master' Execute(status_cmd) def restart(self, env): import params env.set_params(params) self.configure(env) + Logger.info('Restart Elasticsearch master node') restart_cmd = "service elasticsearch restart" - print 'Restarting the Master' Execute(restart_cmd) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py index 633ddd9311..8aaee75290 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py @@ -28,42 +28,43 @@ class Elasticsearch(Script): def install(self, env): import params env.set_params(params) - Logger.info('Install ES Data Node') + Logger.info('Install Elasticsearch data node') self.install_packages(env) def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) + Logger.info('Configure Elasticsearch data node') slave() def stop(self, env, upgrade_type=None): import params env.set_params(params) + Logger.info('Stop Elasticsearch data node') stop_cmd = "service elasticsearch stop" - print 'Stop the Slave' Execute(stop_cmd) def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) + Logger.info('Start Elasticsearch data node') start_cmd = "service elasticsearch start" - print 'Start the Slave' Execute(start_cmd) def status(self, env): import params env.set_params(params) + Logger.info('Check status of Elasticsearch data node') status_cmd = "service elasticsearch status" - print 'Status of the Slave' Execute(status_cmd) def restart(self, env): import params env.set_params(params) self.configure(env) + Logger.info('Restart Elasticsearch data node') restart_cmd = "service elasticsearch restart" - print 'Restarting the Slave' Execute(restart_cmd) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py index b63c71b830..4adcf43dc5 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py @@ -83,3 +83,10 @@ def yamlify_variables(var) : network_host = config['configurations']['elastic-site']['network_host'] network_publish_host = config['configurations']['elastic-site']['network_publish_host'] + +limits_conf_dir = "/etc/security/limits.d" +limits_conf_file = limits_conf_dir + "/elasticsearch.conf" +elastic_user_nofile_limit = config['configurations']['elastic-env']['elastic_user_nofile_limit'] +elastic_user_nproc_limit = config['configurations']['elastic-env']['elastic_user_nproc_limit'] +elastic_user_memlock_soft_limit = config['configurations']['elastic-env']['elastic_user_memlock_soft_limit'] +elastic_user_memlock_hard_limit = config['configurations']['elastic-env']['elastic_user_memlock_hard_limit'] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py index e84fb010ee..d59954f837 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py @@ -17,14 +17,12 @@ limitations under the License. """ -from __future__ import print_function - import subprocess import sys from resource_management.core.resources.system import Execute from resource_management.libraries.script import Script - +from resource_management.core.logger import Logger class ServiceCheck(Script): def service_check(self, env): @@ -34,7 +32,7 @@ def service_check(self, env): doc = '{"name": "Ambari Smoke test"}' index = "ambari_smoke_test" - print("Running Elastic search service check", file=sys.stdout) + Logger.info("Running Elastic search service check", file=sys.stdout) # Make sure the service is actually up. We can live without everything allocated. # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can @@ -47,7 +45,6 @@ def service_check(self, env): ) # Put a document into a new index. - Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True) # Retrieve the document. Use subprocess because we actually need the results here. @@ -55,7 +52,7 @@ def service_check(self, env): proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = proc.communicate() response_retrieve = stdout - print("Retrieval response is: %s" % response_retrieve) + Logger.info("Retrieval response is: %s" % response_retrieve) expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \ % (index, doc) @@ -64,13 +61,13 @@ def service_check(self, env): proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = proc.communicate() response_delete = stdout - print("Delete index response is: %s" % response_retrieve) + Logger.info("Delete index response is: %s" % response_retrieve) expected_delete = '{"acknowledged":true}' if (expected_retrieve == response_retrieve) and (expected_delete == response_delete): - print("Smoke test able to communicate with Elasticsearch") + Logger.info("Smoke test able to communicate with Elasticsearch") else: - print("Elasticsearch service unable to retrieve document.") + Logger.info("Elasticsearch service unable to retrieve document.") sys.exit(1) exit(0) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py index 15a5cf50f0..3303b18c62 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py @@ -23,18 +23,21 @@ from resource_management.core.source import InlineTemplate from resource_management.core.source import Template from resource_management.core.resources import User +from resource_management.core.logger import Logger def slave(): import params + Logger.info("Creating user: {0}:{1}".format(params.elastic_user, params.elastic_group)) User(params.elastic_user, action = "create", groups = params.elastic_group) + params.path_data = params.path_data.replace('"', '') data_path = params.path_data.replace(' ', '').split(',') data_path[:] = [x.replace('"', '') for x in data_path] - directories = [params.log_dir, params.pid_dir, params.conf_dir] directories = directories + data_path + Logger.info("Creating directories: {0}".format(directories)) Directory(directories, create_parents=True, mode=0755, @@ -48,19 +51,30 @@ def slave(): content=InlineTemplate(params.elastic_env_sh_template) ) - configurations = params.config['configurations']['elastic-site'] - - File("{0}/elasticsearch.yml".format(params.conf_dir), + elastic_site = params.config['configurations']['elastic-site'] + path = "{0}/elasticsearch.yml".format(params.conf_dir) + Logger.info("Cre") + File(path, content=Template( "elasticsearch.slave.yaml.j2", - configurations=configurations), + configurations=elastic_site), owner=params.elastic_user, group=params.elastic_group ) - print "Master sysconfig: /etc/sysconfig/elasticsearch" + Logger.info("Slave sysconfig: /etc/sysconfig/elasticsearch") File(format("/etc/sysconfig/elasticsearch"), owner="root", group="root", content=InlineTemplate(params.sysconfig_template) ) + + elastic_env = params.config['configurations']['elastic-env'] + Logger.info("Slave PAM limits: {0}".format(params.limits_conf_file)) + File(params.limits_conf_file, + content=Template( + 'elasticsearch_limits.conf.j2', + configurations=elastic_env), + owner="root", + group="root" + ) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 new file mode 100644 index 0000000000..99f72e1c2d --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{elastic_user}} - nproc {{elastic_user_nproc_limit}} +{{elastic_user}} - nofile {{elastic_user_nofile_limit}} +{{elastic_user}} soft memlock {{elastic_user_memlock_soft_limit}} +{{elastic_user}} hard memlock {{elastic_user_memlock_hard_limit}} diff --git a/metron-deployment/roles/ambari_common/meta/main.yml b/metron-deployment/roles/ambari_common/meta/main.yml index 6077b5af67..d7e46d6696 100644 --- a/metron-deployment/roles/ambari_common/meta/main.yml +++ b/metron-deployment/roles/ambari_common/meta/main.yml @@ -19,4 +19,3 @@ dependencies: - libselinux-python - epel - ntp - - limits diff --git a/metron-deployment/roles/limits/tasks/main.yml b/metron-deployment/roles/limits/tasks/main.yml deleted file mode 100644 index 27e0ad0587..0000000000 --- a/metron-deployment/roles/limits/tasks/main.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Adjust nproc security limits for ES - pam_limits: - domain: elasticsearch - limit_type: '-' - limit_item: nproc - value: 2048 - -- name: Adjust nofile security limits for ES - pam_limits: - domain: elasticsearch - limit_type: '-' - limit_item: nofile - value: 65536 - -- name: Adjust memlock soft security limits for ES - pam_limits: - domain: elasticsearch - limit_type: soft - limit_item: memlock - value: unlimited - -- name: Adjust memlock hard security limits for ES - pam_limits: - domain: elasticsearch - limit_type: hard - limit_item: memlock - value: unlimited - From 928fa049aa1cd24b2540e33deec43d70ffcc955e Mon Sep 17 00:00:00 2001 From: cstella Date: Tue, 24 Oct 2017 14:43:17 -0400 Subject: [PATCH 27/59] Fixing test. --- .../integration/ElasticsearchSearchIntegrationTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index f3dad6a157..b6abb55d93 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -55,6 +55,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "bro_doc": { * "properties": { * "source:type": { "type": "keyword" }, + * "guid": { "type": "keyword" }, * "ip_src_addr": { "type": "ip" }, * "ip_src_port": { "type": "integer" }, * "long_field": { "type": "long" }, @@ -77,6 +78,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "snort_doc": { * "properties": { * "source:type": { "type": "keyword" }, + * "guid": { "type": "keyword" }, * "ip_src_addr": { "type": "ip" }, * "ip_src_port": { "type": "integer" }, * "long_field": { "type": "long" }, From d524e80dcb5cf9798ffc0b1f8c69f0ec7f22dfee Mon Sep 17 00:00:00 2001 From: cstella Date: Tue, 24 Oct 2017 15:30:39 -0400 Subject: [PATCH 28/59] Fixing tests. --- .../integration/ElasticsearchSearchIntegrationTest.java | 4 ++-- .../apache/metron/indexing/dao/SearchIntegrationTest.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index b6abb55d93..4a619e9704 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -160,8 +160,8 @@ protected void loadTestData() .addMapping("bro_doc", broTypeMappings).get(); es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") .addMapping("snort_doc", snortTypeMappings).get(); - es.getClient().admin().indices().prepareCreate("metaalerts") - .addMapping("metaalert_doc", metaalertTypeMappings).get(); + es.getClient().admin().indices().prepareCreate(MetaAlertDao.METAALERTS_INDEX) + .addMapping(MetaAlertDao.METAALERT_DOC, metaalertTypeMappings).get(); BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); JSONArray broArray = (JSONArray) new JSONParser().parse(broData); diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index df33706d51..5ec1202cff 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -561,7 +561,7 @@ public void returns_column_data_for_multiple_indices() throws Exception { Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point")); Assert.assertEquals(FieldType.TEXT, broTypes.get("bro_field")); Assert.assertEquals(FieldType.TEXT, broTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.TEXT, broTypes.get("guid")); + Assert.assertEquals(FieldType.KEYWORD, broTypes.get("guid")); Map snortTypes = fieldTypes.get("snort"); Assert.assertEquals(12, snortTypes.size()); Assert.assertEquals(FieldType.KEYWORD, snortTypes.get("source:type")); @@ -575,7 +575,7 @@ public void returns_column_data_for_multiple_indices() throws Exception { Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.TEXT, broTypes.get("guid")); + Assert.assertEquals(FieldType.KEYWORD, broTypes.get("guid")); } @Test @@ -610,7 +610,7 @@ public void returns_column_metadata_for_specified_indices() throws Exception { Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); } } From c41962ab577e9268fd4fbee2bfd1f42a9d241718 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 25 Oct 2017 12:41:13 -0400 Subject: [PATCH 29/59] Fixing integration test. --- .../ElasticsearchIndexingIntegrationTest.java | 23 ++++++++++- .../components/ElasticSearchComponent.java | 40 ++++++++++++++----- .../integration/IndexingIntegrationTest.java | 10 ++--- 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java index 4c03526dbf..1efcc390b4 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java @@ -17,6 +17,7 @@ */ package org.apache.metron.elasticsearch.integration; +import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.interfaces.FieldNameConverter; import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; import org.apache.metron.elasticsearch.writer.ElasticsearchFieldNameConverter; @@ -35,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTest { @@ -42,6 +44,20 @@ public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTes private String dateFormat = "yyyy.MM.dd.HH"; private String index = "yaf_index_" + new SimpleDateFormat(dateFormat).format(new Date()); private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter(); + /** + * { + * "yaf_doc": { + * "properties": { + * "source:type": { "type": "keyword" }, + * "guid": { "type": "keyword" }, + * "isn": { "type": "text" } + * } + * } + * } + */ + @Multiline + private static String mapping; + @Override public FieldNameConverter getFieldNameConverter() { @@ -53,6 +69,7 @@ public InMemoryComponent getSearchComponent(final Properties topologyProperties) return new ElasticSearchComponent.Builder() .withHttpPort(9211) .withIndexDir(new File(indexDir)) + .withMapping(index, "yaf_doc", mapping) .build(); } @@ -61,6 +78,7 @@ public Processor>> getProcessor(final List inpu return new Processor>>() { List> docs = null; List errors = null; + final AtomicInteger missCount = new AtomicInteger(0); @Override public ReadinessState process(ComponentRunner runner) { ElasticSearchComponent elasticSearchComponent = runner.getComponent("search", ElasticSearchComponent.class); @@ -70,7 +88,9 @@ public ReadinessState process(ComponentRunner runner) { try { docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc"); docsFromDisk = readDocsFromDisk(hdfsDir); - System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size()); + if(missCount.incrementAndGet() >= NUM_RETRIES/2) { + System.out.println(missCount.get() + ": " + docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size()); + } } catch (IOException e) { throw new IllegalStateException("Unable to retrieve indexed documents.", e); } @@ -84,7 +104,6 @@ public ReadinessState process(ComponentRunner runner) { return ReadinessState.READY; } } else { - System.out.println("Missed index..."); return ReadinessState.NOT_READY; } } diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java index e6c5512f29..d0b5980291 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java @@ -22,12 +22,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; + import org.apache.commons.io.FileUtils; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; @@ -56,11 +52,29 @@ public class ElasticSearchComponent implements InMemoryComponent { + private static class Mapping { + String index; + String docType; + String mapping; + + public Mapping(String index, String docType, String mapping) { + this.index = index; + this.docType = docType; + this.mapping = mapping; + } + } + public static class Builder { private int httpPort; private File indexDir; private Map extraElasticSearchSettings = null; + private List mappings = new ArrayList<>(); + + public Builder withMapping(String index, String docType, String mapping) { + mappings.add(new Mapping(index, docType, mapping)); + return this; + } public Builder withHttpPort(int httpPort) { this.httpPort = httpPort; @@ -79,7 +93,7 @@ public Builder withExtraElasticSearchSettings( } public ElasticSearchComponent build() { - return new ElasticSearchComponent(httpPort, indexDir, extraElasticSearchSettings); + return new ElasticSearchComponent(httpPort, indexDir, extraElasticSearchSettings, mappings); } } @@ -89,16 +103,18 @@ public ElasticSearchComponent build() { private int httpPort; private File indexDir; private Map extraElasticSearchSettings; + private List mappings; - public ElasticSearchComponent(int httpPort, File indexDir) { - this(httpPort, indexDir, null); + public ElasticSearchComponent(int httpPort, File indexDir, List mappings) { + this(httpPort, indexDir, null, Collections.EMPTY_LIST); } public ElasticSearchComponent(int httpPort, File indexDir, - Map extraElasticSearchSettings) { + Map extraElasticSearchSettings, List mappings) { this.httpPort = httpPort; this.indexDir = indexDir; this.extraElasticSearchSettings = extraElasticSearchSettings; + this.mappings = mappings; } @Override @@ -132,6 +148,10 @@ public void start() throws UnableToStartException { throw new UnableToStartException("Error starting ES node.", e); } waitForCluster(client, ClusterHealthStatus.YELLOW, STARTUP_TIMEOUT); + for(Mapping m : Optional.ofNullable(mappings).orElse(new ArrayList<>())) { + client.admin().indices().prepareCreate(m.index) + .addMapping(m.docType, m.mapping).get(); + } } private void cleanDir(File dir) throws IOException { diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java index c0f9919f25..ac6f90a60c 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java @@ -56,8 +56,8 @@ public abstract class IndexingIntegrationTest extends BaseIntegrationTest { protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed"; protected String fluxPath = "../metron-indexing/src/main/flux/indexing/remote.yaml"; protected String testSensorType = "test"; - - + protected final int NUM_RETRIES = 100; + protected final long TOTAL_TIME_MS = 150000L; public static List> readDocsFromDisk(String hdfsDirStr) throws IOException { List> ret = new ArrayList<>(); File hdfsDir = new File(hdfsDirStr); @@ -180,8 +180,8 @@ public void test() throws Exception { .withComponent("storm", fluxComponent) .withComponent("search", getSearchComponent(topologyProperties)) .withMillisecondsBetweenAttempts(1500) - .withNumRetries(100) - .withMaxTimeMS(150000) + .withNumRetries(NUM_RETRIES) + .withMaxTimeMS(TOTAL_TIME_MS) .withCustomShutdownOrder(new String[] {"search","storm","config","kafka","zk"}) .build(); @@ -198,8 +198,6 @@ public void test() throws Exception { // on the field name converter assertInputDocsMatchOutputs(inputDocs, docs, getFieldNameConverter()); assertInputDocsMatchOutputs(inputDocs, readDocsFromDisk(hdfsDir), x -> x); - } catch(Throwable e) { - e.printStackTrace(); } finally { if(runner != null) { From 43dfec70afad8e03c839009970c49bb1d71c2517 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 25 Oct 2017 13:24:01 -0400 Subject: [PATCH 30/59] unused constructor. --- .../integration/components/ElasticSearchComponent.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java index d0b5980291..18fe034ca0 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java @@ -105,10 +105,6 @@ public ElasticSearchComponent build() { private Map extraElasticSearchSettings; private List mappings; - public ElasticSearchComponent(int httpPort, File indexDir, List mappings) { - this(httpPort, indexDir, null, Collections.EMPTY_LIST); - } - public ElasticSearchComponent(int httpPort, File indexDir, Map extraElasticSearchSettings, List mappings) { this.httpPort = httpPort; From aabd6b5d0f12f568ce1244f4c974c927b9ca8ff8 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 25 Oct 2017 14:23:01 -0400 Subject: [PATCH 31/59] Updating. --- metron-platform/metron-solr/pom.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml index 487faa102d..a2cf59e7f6 100644 --- a/metron-platform/metron-solr/pom.xml +++ b/metron-platform/metron-solr/pom.xml @@ -65,6 +65,18 @@ + + org.apache.logging.log4j + log4j-api + ${global_log4j_core_version} + test + + + org.apache.logging.log4j + log4j-core + ${global_log4j_core_version} + test + org.apache.storm storm-core From f11c2333c669ce9924b1bcc922ead7ec562f7314 Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 26 Oct 2017 15:47:25 -0400 Subject: [PATCH 32/59] Correcting a few heinous bugs. --- .../METRON/CURRENT/package/scripts/indexing_master.py | 11 +++++++---- .../metron/rest/controller/RestExceptionHandler.java | 5 +++++ .../metron/rest/service/impl/UpdateServiceImpl.java | 6 +++++- .../metron/elasticsearch/dao/ElasticsearchDao.java | 5 +++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 086cab0332..2330d24b3b 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -144,9 +144,12 @@ def elasticsearch_template_install(self, env): error_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/error_index -d @{error_index_path}') Execute(error_cmd, logoutput=True) - error_cmd = ambari_format( + metaalert_template_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/metaalert_index -d @{meta_index_path}') - Execute(error_cmd, logoutput=True) + Execute(metaalert_template_cmd, logoutput=True) + metaalert_index_cmd = ambari_format( 'curl -s -XPUT http://{es_http_url}/metaalert_index') + Execute(metaalert_index_cmd, logoutput=True) + def elasticsearch_template_delete(self, env): from params import params @@ -160,8 +163,8 @@ def elasticsearch_template_delete(self, env): Execute(yaf_cmd, logoutput=True) error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/error_index*"') Execute(error_cmd, logoutput=True) - error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/metaalerts*"') - Execute(error_cmd, logoutput=True) + metaalert_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/metaalert_index"') + Execute(metaalert_cmd, logoutput=True) def zeppelin_notebook_import(self, env): from params import params diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java index 5e6f7e741b..a3bab30343 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java @@ -20,6 +20,8 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.RestError; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ControllerAdvice; @@ -28,14 +30,17 @@ import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; import javax.servlet.http.HttpServletRequest; +import java.lang.invoke.MethodHandles; @ControllerAdvice(basePackages = "org.apache.metron.rest.controller") public class RestExceptionHandler extends ResponseEntityExceptionHandler { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @ExceptionHandler(RestException.class) @ResponseBody ResponseEntity handleControllerException(HttpServletRequest request, Throwable ex) { HttpStatus status = getStatus(request); + LOG.error("Encountered error: " + ex.getMessage(), ex); return new ResponseEntity<>(new RestError(status.value(), ex.getMessage(), ExceptionUtils.getRootCauseMessage(ex)), status); } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java index 847173e716..76ac75db6a 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java @@ -23,15 +23,18 @@ import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.UpdateService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.lang.invoke.MethodHandles; import java.util.Optional; @Service public class UpdateServiceImpl implements UpdateService { private IndexDao dao; - + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Autowired public UpdateServiceImpl(IndexDao dao) { this.dao = dao; @@ -43,6 +46,7 @@ public void patch(PatchRequest request) throws RestException, OriginalNotFoundEx try { dao.patch(request, Optional.of(System.currentTimeMillis())); } catch (Exception e) { + throw new RestException(e.getMessage(), e); } } diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index f9646a6138..0da87688ce 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -252,6 +252,9 @@ public Document getLatest(final String guid, final String sensorType) throws IOE */ Optional searchByGuid(String guid, String sensorType, Function> callback) { + if(guid == null) { + return Optional.empty(); + } QueryBuilder query = QueryBuilders.idsQuery(sensorType + "_doc").addIds(guid); SearchRequestBuilder request = client.prepareSearch() .setQuery(query) @@ -300,8 +303,6 @@ public void update(Document update, Optional index) throws IOException { .upsert(indexRequest) ; - org.elasticsearch.action.search.SearchResponse result = client.prepareSearch("test*").setFetchSource(true).setQuery(QueryBuilders.matchAllQuery()).get(); - result.getHits(); try { UpdateResponse response = client.update(updateRequest).get(); From 8828c235c8d01c7d13bed45f712dc72552f5ea9b Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Thu, 9 Nov 2017 22:04:21 -0700 Subject: [PATCH 33/59] Get Kibana working. Rewrite Kibana dashboard from scratch and deploy via ES bulk JSON REST call. --- .../KIBANA/5.6.2/configuration/kibana-env.xml | 5 + .../5.6.2/configuration/kibana-site.xml | 3 +- .../scripts/dashboard/dashboard-bulkload.json | 64 + .../package/scripts/dashboard/dashboard.p | 2341 ----------------- .../5.6.2/package/scripts/kibana_master.py | 23 +- .../KIBANA/5.6.2/package/scripts/params.py | 2 +- .../bulk/ElasticsearchImportExport.java | 76 + .../bulk/ElasticsearchImportExportTest.java | 69 + .../metron/integration/utils/TestUtils.java | 27 + pom.xml | 2 - 10 files changed, 253 insertions(+), 2359 deletions(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json delete mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard.p create mode 100644 metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java create mode 100644 metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml index 0adf32ad7a..1246405813 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml @@ -39,6 +39,11 @@ false + + kibana_server_host + 0.0.0.0 + Host name or IP address that Kibana should bind to. + kibana_log_dir /var/log/kibana diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml index 4373d140f9..d8d05135ea 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml @@ -29,7 +29,8 @@ server.port: {{ kibana_port }} # The host to bind the server to. -# server.host: "0.0.0.0" +# Kibana (like Elasticsearch) now binds to localhost for security purposes instead of 0.0.0.0 (all addresses). Previous binding to 0.0.0.0 also caused issues for Windows users. +server.host: {{ kibana_server_host }} # If you are running kibana behind a proxy, and want to mount it at a path, # specify that path here. The basePath can't end in a slash. diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json new file mode 100644 index 0000000000..dfadd48a00 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json @@ -0,0 +1,64 @@ +{ "create" : { "_id": "all-metron-index", "_type": "index-pattern" } } +{"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:ca\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:path_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bro_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bro_timestamp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"capture_password\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:exponent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:issuer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_after\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_before\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:sig_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"client\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"compression_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_uids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"connect_info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_channel:orig_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dgmlen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dhcp_host_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dropped\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dst\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"end_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"end_reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_hash\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"established\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethdst\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethlen\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethsrc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"exception\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failed_sensor_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failure_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_desc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"first_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"helo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"history\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"id\",\"type\":\"conflict\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"conflictDescriptions\":{\"integer\":[\"snort_index_2017.11.06.19\",\"snort_index_2017.11.06.20\",\"snort_index_2017.11.06.21\",\"snort_index_2017.11.06.22\",\"snort_index_2017.11.06.23\",\"snort_index_2017.11.07.00\",\"snort_index_2017.11.07.01\"],\"keyword\":[\"bro_index_2017.11.02.23\",\"bro_index_2017.11.03.00\",\"bro_index_2017.11.03.01\",\"bro_index_2017.11.03.02\",\"bro_index_2017.11.03.03\",\"bro_index_2017.11.03.04\",\"bro_index_2017.11.03.13\",\"bro_index_2017.11.06.19\",\"bro_index_2017.11.06.20\",\"bro_index_2017.11.06.22\",\"bro_index_2017.11.06.23\",\"bro_index_2017.11.07.00\",\"bro_index_2017.11.07.01\"]}},{\"name\":\"iflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"in_reply_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"iplen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_alert\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_orig\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_webmail\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"isn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"issuer_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"kex_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_alert\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_reply\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"lease_time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_orig\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_resp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mailfrom\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"md5\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"method\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"missed_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"missing_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"msg_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"n\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"next_protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"note\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"notice\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"oct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"overflow_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"parent_fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"password\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer_descr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"pkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"port_num\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"proto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message_bytes\",\"type\":\"unknown\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"rcode\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcode_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcptto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"referrer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rejected\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"remote_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"request_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"response_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"result\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resumed\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"riflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"risn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"roct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rpkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ruflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:dns\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:email\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"second_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"seen_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sensor:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sensor:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"service\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha1\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha256\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_generator\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_rev\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"sip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"software_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"source:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src_peer\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"stack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sub\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"suppress_for\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tcpack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpseq\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpwindow\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"threat:triage:level\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:rules:0:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timedout\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tls\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tos\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"total_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ttl\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tunnel_parents\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"unparsed_version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"username\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:major\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor2\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor3\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"x_originating_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"} +{ "create" : { "_id": "AV-Sj0e2hKs1cXXnFMqF", "_type": "visualization" } } +{"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-Yh94VdXwc6Ua9Muh0", "_type": "index-pattern" } } +{"title":"error_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"error_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_hash\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"exception\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failed_sensor_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message_bytes\",\"type\":\"unknown\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"stack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"} +{ "index" : { "_id": "5.6.2", "_type": "config" } } +{"defaultIndex":"AV-S2e81hKs1cXXnFMqN"} +{ "create" : { "_id": "AV-dVurck7f2nZ-iH3Ka", "_type": "visualization" } } +{"title":"Event Count By Type","visState":"{\"title\":\"Event Count By Type\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"source:type\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"legendOpen\":true,\"colors\":{\"yaf\":\"#CCA300\",\"snort\":\"#C15C17\",\"bro\":\"#F9934E\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, yaf, snort\",\"params\":[\"bro\",\"yaf\",\"snort\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}},{\"match_phrase\":{\"source:type\":\"snort\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-YyJw3PfR7HJex-ZdY", "_type": "visualization" } } +{"title":"All index TS event count","visState":"{\"title\":\"All index TS event count\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"metric\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Event Count\",\"split_filters\":[{\"color\":\"#68BC00\",\"id\":\"89be23f0-c4af-11e7-ac01-25d5c1ff2e49\"}],\"series_drop_last_bucket\":0}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro_index*,snort_index*,yaf_index*\",\"interval\":\"1y\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0f0-c412-11e7-a0b9-2137696bd057\"}],\"filter\":\"\",\"drop_last_bucket\":0},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-cBm5JFLIoshSSHghu", "_type": "visualization" } } +{"title":"All index TS Chart","visState":"{\"title\":\"All index TS Chart\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"timeseries\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":\"1\",\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Events\",\"terms_field\":\"source:type\",\"value_template\":\"{{value}}\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro*,snort*,yaf*\",\"interval\":\"30s\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0f0-c412-11e7-a0b9-2137696bd057\"}],\"show_grid\":1,\"drop_last_bucket\":0},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-dXz9Lk7f2nZ-iH3Kb", "_type": "visualization" } } +{"title":"Event Count Pie Chart","visState":"{\"title\":\"Event Count Pie Chart\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Events by Source Type\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source:type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-ddhh7k7f2nZ-iH3Kx", "_type": "visualization" } } +{"title":"Flow Location Map","visState":"{\"title\":\"Flow Location Map\",\"type\":\"tile_map\",\"params\":{\"mapType\":\"Scaled Circle Markers\",\"isDesaturated\":true,\"addTooltip\":true,\"heatMaxZoom\":0,\"heatMinOpacity\":0.1,\"heatRadius\":25,\"heatBlur\":15,\"legendPosition\":\"bottomright\",\"mapZoom\":2,\"mapCenter\":[0,0],\"wms\":{\"enabled\":false,\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\"options\":{\"version\":\"1.3.0\",\"layers\":\"0\",\"format\":\"image/png\",\"transparent\":true,\"attribution\":\"Maps provided by USGS\",\"styles\":\"\"}},\"type\":\"tile_map\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:location_point\",\"autoPrecision\":true,\"useGeocentroid\":true,\"precision\":2,\"customLabel\":\"Flow Source Locations\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-dfk_gk7f2nZ-iH3K0", "_type": "visualization" } } +{"title":"Events By Country","visState":"{\"title\":\"Events By Country\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:country\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-TUPlDgto7-W6O2b3n", "_type": "index-pattern" } } +{"title":"yaf_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"end_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"end_reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"iflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"isn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"oct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"pkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"proto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"riflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"risn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"roct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rpkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ruflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"} +{ "create" : { "_id": "AV-eebabk7f2nZ-iH3L1", "_type": "visualization" } } +{"title":"YAF Flow Duration","visState":"{\"title\":\"YAF Flow Duration\",\"type\":\"area\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"Flow Duration (seconds)\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"area\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"duration\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Flow Duration (seconds)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TUPlDgto7-W6O2b3n\",\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-deDqXk7f2nZ-iH3Ky", "_type": "visualization" } } +{"title":"Geo-IP Locations","visState":"{\"title\":\"Geo-IP Locations\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"60\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:country\",\"customLabel\":\"Unique Location(s)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-YvG0DPfR7HJex-ZaS", "_type": "visualization" } } +{"title":"Event Count","visState":"{\"title\":\"Event Count\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"fontSize\":\"60\",\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Event Count\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-ejKEdk7f2nZ-iH3MI", "_type": "visualization" } } +{"title":"Web Requests","visState":"{\"title\":\"Web Requests\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"type\":\"phrases\",\"key\":\"protocol\",\"value\":\"http, https\",\"params\":[\"http\",\"https\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"protocol\":\"http\"}},{\"match_phrase\":{\"protocol\":\"https\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-ejbG6k7f2nZ-iH3MJ", "_type": "visualization" } } +{"title":"DNS Requests","visState":"{\"title\":\"DNS Requests\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"protocol\",\"value\":\"dns\"},\"query\":{\"match\":{\"protocol\":{\"query\":\"dns\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-eh5Wgk7f2nZ-iH3MG", "_type": "visualization" } } +{"title":"Snort Alert Types","visState":"{\"title\":\"Snort Alert Types\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sig_id\",\"customLabel\":\"Alert Type(s)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TAoyPhKs1cXXnFMqi\",\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-ecrFkk7f2nZ-iH3L0", "_type": "visualization" } } +{"title":"Yaf Flows Count","visState":"{\"title\":\"Yaf Flows Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TUPlDgto7-W6O2b3n\",\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-ek_Jnk7f2nZ-iH3MK", "_type": "visualization" } } +{"title":"Web Request Type","visState":"{\"title\":\"Web Request Type\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"method\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"type\":\"phrases\",\"key\":\"protocol\",\"value\":\"http, https\",\"params\":[\"http\",\"https\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"protocol\":\"http\"}},{\"match_phrase\":{\"protocol\":\"https\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-S2e81hKs1cXXnFMqN", "_type": "index-pattern" } } +{"title":"bro_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:ca\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:path_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bro_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bro_timestamp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"capture_password\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:exponent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:issuer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_after\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_before\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:sig_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"client\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"compression_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_uids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"connect_info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_channel:orig_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dhcp_host_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dropped\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dst\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"established\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failure_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_desc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"first_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"helo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"history\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"in_reply_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_orig\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_webmail\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"issuer_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"kex_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_alert\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_reply\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"lease_time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_orig\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_resp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mailfrom\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"md5\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"method\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"missed_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"missing_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"msg\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"msg_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"n\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"next_protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"note\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"notice\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"overflow_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"parent_fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"password\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer_descr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"port_num\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"proto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype_name\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"query\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcode\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcode_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcptto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"referrer\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rejected\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"remote_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"request_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"response_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"result\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resumed\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:dns\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:email\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"second_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"seen_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"service\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha1\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha256\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"software_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src_peer\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sub\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"suppress_for\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timedout\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tls\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"total_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tunnel_parents\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"unparsed_version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"uri\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"username\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:major\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor2\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor3\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"x_originating_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"} +{ "create" : { "_id": "AV-TAoyPhKs1cXXnFMqi", "_type": "index-pattern" } } +{"title":"snort_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dgmlen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethdst\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethlen\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethsrc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"iplen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_alert\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"msg\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_generator\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_id\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_rev\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tcpack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpseq\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpwindow\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"threat:triage:level\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:rules:0:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tos\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ttl\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"} +{ "create" : { "_id": "AV-YpDmwdXwc6Ua9Muh9", "_type": "dashboard" } } +{"title":"Metron-Dashboard","hits":0,"description":"","panelsJSON":"[{\"col\":1,\"id\":\"AV-Sj0e2hKs1cXXnFMqF\",\"panelIndex\":1,\"row\":1,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"id\":\"AV-cBm5JFLIoshSSHghu\",\"panelIndex\":3,\"row\":4,\"size_x\":9,\"size_y\":6,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-YvG0DPfR7HJex-ZaS\",\"panelIndex\":4,\"row\":4,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-dXz9Lk7f2nZ-iH3Kb\",\"panelIndex\":5,\"row\":7,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-dYnlek7f2nZ-iH3Kc\",\"panelIndex\":6,\"row\":10,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-deDqXk7f2nZ-iH3Ky\",\"panelIndex\":7,\"row\":12,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-dfk_gk7f2nZ-iH3K0\",\"panelIndex\":8,\"row\":15,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"AV-ddhh7k7f2nZ-iH3Kx\",\"panelIndex\":9,\"row\":10,\"size_x\":9,\"size_y\":7,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-dYtN5k7f2nZ-iH3Kd\",\"panelIndex\":10,\"row\":17,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-dYxfHk7f2nZ-iH3Ke\",\"panelIndex\":11,\"row\":24,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-dY9zbk7f2nZ-iH3Kf\",\"panelIndex\":12,\"row\":31,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-dZKsfk7f2nZ-iH3Kg\",\"panelIndex\":13,\"row\":38,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"ip_src_addr\",\"ip_src_port\",\"ip_dst_addr\",\"ip_dst_port\",\"protocol\"],\"id\":\"AV-eSl9lk7f2nZ-iH3Lj\",\"panelIndex\":14,\"row\":17,\"size_x\":9,\"size_y\":7,\"sort\":[\"timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"AV-ecrFkk7f2nZ-iH3L0\",\"panelIndex\":15,\"row\":19,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-eebabk7f2nZ-iH3L1\",\"panelIndex\":16,\"row\":22,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-eh5Wgk7f2nZ-iH3MG\",\"panelIndex\":17,\"row\":26,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-ejKEdk7f2nZ-iH3MI\",\"panelIndex\":18,\"row\":33,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV-ejbG6k7f2nZ-iH3MJ\",\"panelIndex\":19,\"row\":40,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"msg\",\"sig_id\",\"ip_src_addr\",\"ip_src_port\",\"ip_dst_addr\",\"ip_dst_port\",\"protocol\"],\"id\":\"AV-eloCYk7f2nZ-iH3ML\",\"panelIndex\":20,\"row\":24,\"size_x\":9,\"size_y\":7,\"sort\":[\"timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":4,\"columns\":[\"method\",\"host\",\"uri\",\"referrer\"],\"id\":\"AV-eltB3k7f2nZ-iH3MM\",\"panelIndex\":21,\"row\":31,\"size_x\":9,\"size_y\":7,\"sort\":[\"timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":4,\"columns\":[\"query\",\"qtype_name\",\"answers\",\"ip_src_addr\",\"ip_dst_addr\"],\"id\":\"AV-emMQYk7f2nZ-iH3MN\",\"panelIndex\":22,\"row\":38,\"size_x\":9,\"size_y\":7,\"sort\":[\"timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"AV-ek_Jnk7f2nZ-iH3MK\",\"panelIndex\":23,\"row\":36,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"size_x\":3,\"size_y\":2,\"panelIndex\":24,\"type\":\"visualization\",\"id\":\"AV-eoTnqk7f2nZ-iH3MO\",\"col\":1,\"row\":29}]","optionsJSON":"{\"darkTheme\":false}","uiStateJSON":"{\"P-4\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}}},\"P-7\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-15\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-17\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-18\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-19\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-24\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}","version":1,"timeRestore":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"}} +{ "create" : { "_id": "AV-eoTnqk7f2nZ-iH3MO", "_type": "visualization" } } +{"title":"Snort Top Alerts By Host","visState":"{\"title\":\"Snort Top Alerts By Host\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_src_addr\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_dst_addr\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Destination\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TAoyPhKs1cXXnFMqi\",\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-eltB3k7f2nZ-iH3MM", "_type": "search" } } +{"title":"Web Requests","description":"","hits":0,"columns":["method","host","uri","referrer"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"highlightAll\":true,\"version\":true,\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"type\":\"phrases\",\"key\":\"protocol\",\"value\":\"http, https\",\"params\":[\"http\",\"https\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"protocol\":\"http\"}},{\"match_phrase\":{\"protocol\":\"https\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-emMQYk7f2nZ-iH3MN", "_type": "search" } } +{"title":"DNS Requests","description":"","hits":0,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"highlightAll\":true,\"version\":true,\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"protocol\",\"value\":\"dns\"},\"query\":{\"match\":{\"protocol\":{\"query\":\"dns\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"}} +{ "create" : { "_id": "AV-eloCYk7f2nZ-iH3ML", "_type": "search" } } +{"title":"Snort Alerts","description":"","hits":0,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TAoyPhKs1cXXnFMqi\",\"highlightAll\":true,\"version\":true,\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-eSl9lk7f2nZ-iH3Lj", "_type": "search" } } +{"title":"YAF Flows","description":"","hits":0,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TUPlDgto7-W6O2b3n\",\"highlightAll\":true,\"version\":true,\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-dZKsfk7f2nZ-iH3Kg", "_type": "visualization" } } +{"title":"DNS Requests Overview","visState":"{\"title\":\"DNS Requests Overview\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-dYnlek7f2nZ-iH3Kc", "_type": "visualization" } } +{"title":"Enrichment Overview","visState":"{\"title\":\"Enrichment Overview\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-dYxfHk7f2nZ-iH3Ke", "_type": "visualization" } } +{"title":"Snort Overview","visState":"{\"title\":\"Snort Overview\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-dY9zbk7f2nZ-iH3Kf", "_type": "visualization" } } +{"title":"Web Request Header Overview","visState":"{\"title\":\"Web Request Header Overview\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{ "create" : { "_id": "AV-dYtN5k7f2nZ-iH3Kd", "_type": "visualization" } } +{"title":"YAF Overview","visState":"{\"title\":\"YAF Overview\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard.p b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard.p deleted file mode 100644 index efff33d42f..0000000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard.p +++ /dev/null @@ -1,2341 +0,0 @@ -(lp1 -(dp2 -V_score -p3 -F1 -sV_type -p4 -Vindex-pattern -p5 -sV_id -p6 -Vbro* -p7 -sV_source -p8 -(dp9 -Vfields -p10 -V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p11 -sVtimeFieldName -p12 -Vtimestamp -p13 -sVtitle -p14 -Vbro* -p15 -ssV_index -p16 -V.kibana -p17 -sa(dp18 -V_score -p19 -F1 -sV_type -p20 -Vsearch -p21 -sV_id -p22 -Vsnort-search -p23 -sV_source -p24 -(dp25 -Vsort -p26 -(lp27 -Vtimestamp -p28 -aVdesc -p29 -asVhits -p30 -I0 -sVdescription -p31 -V -sVtitle -p32 -VSnort Alerts -p33 -sVversion -p34 -I1 -sVkibanaSavedObjectMeta -p35 -(dp36 -VsearchSourceJSON -p37 -V{"index":"snort*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p38 -ssVcolumns -p39 -(lp40 -Vmsg -p41 -aVsig_id -p42 -aVip_src_addr -p43 -aVip_src_port -p44 -aVip_dst_addr -p45 -aVip_dst_port -p46 -assV_index -p47 -V.kibana -p48 -sa(dp49 -V_score -p50 -F1 -sV_type -p51 -Vsearch -p52 -sV_id -p53 -Vyaf-search -p54 -sV_source -p55 -(dp56 -Vsort -p57 -(lp58 -Vtimestamp -p59 -aVdesc -p60 -asVhits -p61 -I0 -sVdescription -p62 -V -sVtitle -p63 -VYAF -p64 -sVversion -p65 -I1 -sVkibanaSavedObjectMeta -p66 -(dp67 -VsearchSourceJSON -p68 -V{"index":"yaf*","filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647},"query":{"query_string":{"query":"*","analyze_wildcard":true}}} -p69 -ssVcolumns -p70 -(lp71 -Vip_src_addr -p72 -aVip_src_port -p73 -aVip_dst_addr -p74 -aVip_dst_port -p75 -aVprotocol -p76 -aVduration -p77 -aVpkt -p78 -assV_index -p79 -V.kibana -p80 -sa(dp81 -V_score -p82 -F1 -sV_type -p83 -Vvisualization -p84 -sV_id -p85 -VWelcome -p86 -sV_source -p87 -(dp88 -VvisState -p89 -V{"title":"Welcome to Apache Metron","type":"markdown","params":{"markdown":"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\u005cn\u005cnApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\u005cn\u005cnThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron."},"aggs":[],"listeners":{}} -p90 -sVdescription -p91 -V -sVtitle -p92 -VWelcome to Apache Metron -p93 -sVuiStateJSON -p94 -V{} -p95 -sVversion -p96 -I1 -sVkibanaSavedObjectMeta -p97 -(dp98 -VsearchSourceJSON -p99 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p100 -sssV_index -p101 -V.kibana -p102 -sa(dp103 -V_score -p104 -F1 -sV_type -p105 -Vvisualization -p106 -sV_id -p107 -VTop-Snort-Alerts-by-Source -p108 -sV_source -p109 -(dp110 -VvisState -p111 -V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}} -p112 -sVdescription -p113 -V -sVtitle -p114 -VTop Snort Alerts by Source -p115 -sVuiStateJSON -p116 -V{} -p117 -sVversion -p118 -I1 -sVkibanaSavedObjectMeta -p119 -(dp120 -VsearchSourceJSON -p121 -V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p122 -sssV_index -p123 -V.kibana -p124 -sa(dp125 -V_score -p126 -F1 -sV_type -p127 -Vvisualization -p128 -sV_id -p129 -VWeb-Request-Type -p130 -sV_source -p131 -(dp132 -VvisState -p133 -V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p134 -sVdescription -p135 -V -sVtitle -p136 -VWeb Request Type -p137 -sVuiStateJSON -p138 -V{} -p139 -sVversion -p140 -I1 -sVsavedSearchId -p141 -Vweb-search -p142 -sVkibanaSavedObjectMeta -p143 -(dp144 -VsearchSourceJSON -p145 -V{"filter":[]} -p146 -sssV_index -p147 -V.kibana -p148 -sa(dp149 -V_score -p150 -F1 -sV_type -p151 -Vconfig -p152 -sV_id -p153 -V4.5.1 -p154 -sV_source -p155 -(dp156 -VbuildNum -p157 -I9892 -sVdefaultIndex -p158 -Vbro* -p159 -ssV_index -p160 -V.kibana -p161 -sa(dp162 -V_score -p163 -F1 -sV_type -p164 -Vvisualization -p165 -sV_id -p166 -VErrors-By-Hostname -p167 -sV_source -p168 -(dp169 -VvisState -p170 -V{\u000a "title": "Errors By Error Type",\u000a "type": "histogram",\u000a "params": {\u000a "addLegend": true,\u000a "addTimeMarker": false,\u000a "addTooltip": true,\u000a "defaultYExtents": false,\u000a "mode": "grouped",\u000a "scale": "linear",\u000a "setYExtents": false,\u000a "shareYAxis": true,\u000a "times": [],\u000a "yAxis": {}\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {\u000a "customLabel": "Count"\u000a }\u000a },\u000a {\u000a "id": "2",\u000a "type": "terms",\u000a "schema": "segment",\u000a "params": {\u000a "field": "hostname",\u000a "size": 5,\u000a "order": "desc",\u000a "orderBy": "1"\u000a }\u000a },\u000a {\u000a "id": "4",\u000a "type": "cardinality",\u000a "schema": "metric",\u000a "params": {\u000a "field": "error_hash",\u000a "customLabel": "Unique Datapoint Count"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} -p171 -sVdescription -p172 -V -sVtitle -p173 -VErrors By Hostname -p174 -sVuiStateJSON -p175 -V{\u000a "vis": {\u000a "colors": {\u000a "Unique Datapoint Count": "#9AC48A",\u000a "Count": "#629E51"\u000a }\u000a }\u000a} -p176 -sVversion -p177 -I1 -sVkibanaSavedObjectMeta -p178 -(dp179 -VsearchSourceJSON -p180 -V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "analyze_wildcard": true,\u000a "query": "*"\u000a }\u000a },\u000a "filter": []\u000a} -p181 -sssV_index -p182 -V.kibana -p183 -sa(dp184 -V_score -p185 -F1 -sV_type -p186 -Vvisualization -p187 -sV_id -p188 -VWeb-Request-Header -p189 -sV_source -p190 -(dp191 -VvisState -p192 -V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}} -p193 -sVdescription -p194 -V -sVtitle -p195 -VWeb Request Header -p196 -sVuiStateJSON -p197 -V{} -p198 -sVversion -p199 -I1 -sVkibanaSavedObjectMeta -p200 -(dp201 -VsearchSourceJSON -p202 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p203 -sssV_index -p204 -V.kibana -p205 -sa(dp206 -V_score -p207 -F1 -sV_type -p208 -Vvisualization -p209 -sV_id -p210 -VError-Type-Proportion -p211 -sV_source -p212 -(dp213 -VvisState -p214 -V{"title":"Error Type Proportion","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"error_type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p215 -sVdescription -p216 -V -sVtitle -p217 -VError Type Proportion -p218 -sVuiStateJSON -p219 -V{} -p220 -sVversion -p221 -I1 -sVkibanaSavedObjectMeta -p222 -(dp223 -VsearchSourceJSON -p224 -V{"index":"error*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p225 -sssV_index -p226 -V.kibana -p227 -sa(dp228 -V_score -p229 -F1 -sV_type -p230 -Vvisualization -p231 -sV_id -p232 -VFlow-Duration -p233 -sV_source -p234 -(dp235 -VvisState -p236 -V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}} -p237 -sVdescription -p238 -V -sVtitle -p239 -VFlow Duration -p240 -sVuiStateJSON -p241 -V{"vis":{"legendOpen":false}} -p242 -sVversion -p243 -I1 -sVkibanaSavedObjectMeta -p244 -(dp245 -VsearchSourceJSON -p246 -V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p247 -sssV_index -p248 -V.kibana -p249 -sa(dp250 -V_score -p251 -F1 -sV_type -p252 -Vvisualization -p253 -sV_id -p254 -VErrors-By-Source -p255 -sV_source -p256 -(dp257 -VvisState -p258 -V{"title":"Errors By Source","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source_type","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}}],"listeners":{}} -p259 -sVdescription -p260 -V -sVtitle -p261 -VErrors By Source -p262 -sVuiStateJSON -p263 -V{} -p264 -sVversion -p265 -I1 -sVkibanaSavedObjectMeta -p266 -(dp267 -VsearchSourceJSON -p268 -V{"index":"error*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p269 -sssV_index -p270 -V.kibana -p271 -sa(dp272 -V_score -p273 -F1 -sV_type -p274 -Vvisualization -p275 -sV_id -p276 -VEvents -p277 -sV_source -p278 -(dp279 -VvisState -p280 -V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p281 -sVdescription -p282 -V -sVtitle -p283 -VEvents -p284 -sVuiStateJSON -p285 -V{"vis":{"legendOpen":false}} -p286 -sVversion -p287 -I1 -sVkibanaSavedObjectMeta -p288 -(dp289 -VsearchSourceJSON -p290 -V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p291 -sssV_index -p292 -V.kibana -p293 -sa(dp294 -V_score -p295 -F1 -sV_type -p296 -Vvisualization -p297 -sV_id -p298 -VError-Hostname-Proportion -p299 -sV_source -p300 -(dp301 -VvisState -p302 -V{"aggs":[{"id":"1","params":{},"schema":"metric","type":"count"},{"id":"2","params":{"customLabel":"Sensor","field":"hostname","order":"desc","orderBy":"1","size":5},"schema":"segment","type":"terms"}],"listeners":{},"params":{"addLegend":true,"addTooltip":true,"isDonut":false,"shareYAxis":true},"title":"Error Source Proportion","type":"pie"} -p303 -sVdescription -p304 -V -sVtitle -p305 -VError Hostname Proportion -p306 -sVuiStateJSON -p307 -V{"vis":{"colors":{"host":"#629E51","host2":"#9AC48A","hostAnother":"#7EB26D","hostNew":"#B7DBAB"}}} -p308 -sVversion -p309 -I1 -sVkibanaSavedObjectMeta -p310 -(dp311 -VsearchSourceJSON -p312 -V{"index":"error*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p313 -sssV_index -p314 -V.kibana -p315 -sa(dp316 -V_score -p317 -F1 -sV_type -p318 -Vvisualization -p319 -sV_id -p320 -VUnique-Error-Messages -p321 -sV_source -p322 -(dp323 -VvisState -p324 -V{\u000a "title": "Total Unique Error Messages",\u000a "type": "metric",\u000a "params": {\u000a "handleNoResults": true,\u000a "fontSize": 60\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "cardinality",\u000a "schema": "metric",\u000a "params": {\u000a "field": "error_hash",\u000a "customLabel": "Unique Error Messages"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} -p325 -sVdescription -p326 -V -sVtitle -p327 -VUnique Error Messages -p328 -sVuiStateJSON -p329 -V{} -p330 -sVversion -p331 -I1 -sVkibanaSavedObjectMeta -p332 -(dp333 -VsearchSourceJSON -p334 -V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "query": "*",\u000a "analyze_wildcard": true\u000a }\u000a },\u000a "filter": []\u000a} -p335 -sssV_index -p336 -V.kibana -p337 -sa(dp338 -V_score -p339 -F1 -sV_type -p340 -Vvisualization -p341 -sV_id -p342 -VErrors-By-Error-Type -p343 -sV_source -p344 -(dp345 -VvisState -p346 -V{\u000a "title": "Errors By Error Type",\u000a "type": "histogram",\u000a "params": {\u000a "addLegend": true,\u000a "addTimeMarker": false,\u000a "addTooltip": true,\u000a "defaultYExtents": false,\u000a "mode": "grouped",\u000a "scale": "linear",\u000a "setYExtents": false,\u000a "shareYAxis": true,\u000a "times": [],\u000a "yAxis": {}\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {\u000a "customLabel": "Count"\u000a }\u000a },\u000a {\u000a "id": "2",\u000a "type": "terms",\u000a "schema": "segment",\u000a "params": {\u000a "field": "error_type",\u000a "size": 5,\u000a "order": "desc",\u000a "orderBy": "1"\u000a }\u000a },\u000a {\u000a "id": "4",\u000a "type": "cardinality",\u000a "schema": "metric",\u000a "params": {\u000a "field": "error_hash",\u000a "customLabel": "Unique Datapoint Count"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} -p347 -sVdescription -p348 -V -sVtitle -p349 -VErrors By Error Type -p350 -sVuiStateJSON -p351 -V{\u000a "vis": {\u000a "colors": {\u000a "Unique Datapoint Count": "#806EB7",\u000a "Count": "#614D93"\u000a }\u000a }\u000a} -p352 -sVversion -p353 -I1 -sVkibanaSavedObjectMeta -p354 -(dp355 -VsearchSourceJSON -p356 -V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "analyze_wildcard": true,\u000a "query": "*"\u000a }\u000a },\u000a "filter": []\u000a} -p357 -sssV_index -p358 -V.kibana -p359 -sa(dp360 -V_score -p361 -F1 -sV_type -p362 -Vsearch -p363 -sV_id -p364 -VErrors -p365 -sV_source -p366 -(dp367 -Vsort -p368 -(lp369 -Vtimestamp -p370 -aVdesc -p371 -asVhits -p372 -I0 -sVdescription -p373 -V -sVtitle -p374 -VErrors -p375 -sVversion -p376 -I1 -sVkibanaSavedObjectMeta -p377 -(dp378 -VsearchSourceJSON -p379 -V{"index":"error*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p380 -ssVcolumns -p381 -(lp382 -Vfailed_sensor_type -p383 -aVerror_type -p384 -aVexception -p385 -aVhostname -p386 -aVmessage -p387 -aVraw_message -p388 -aVerror_hash -p389 -assV_index -p390 -V.kibana -p391 -sa(dp392 -V_score -p393 -F1 -sV_type -p394 -Vvisualization -p395 -sV_id -p396 -VSnort-Header -p397 -sV_source -p398 -(dp399 -VvisState -p400 -V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}} -p401 -sVdescription -p402 -V -sVtitle -p403 -VSnort -p404 -sVuiStateJSON -p405 -V{} -p406 -sVversion -p407 -I1 -sVkibanaSavedObjectMeta -p408 -(dp409 -VsearchSourceJSON -p410 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p411 -sssV_index -p412 -V.kibana -p413 -sa(dp414 -V_score -p415 -F1 -sV_type -p416 -Vvisualization -p417 -sV_id -p418 -VYAF-Flow(s) -p419 -sV_source -p420 -(dp421 -VvisState -p422 -V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p423 -sVdescription -p424 -V -sVtitle -p425 -VYAF Flows -p426 -sVuiStateJSON -p427 -V{} -p428 -sVversion -p429 -I1 -sVkibanaSavedObjectMeta -p430 -(dp431 -VsearchSourceJSON -p432 -V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p433 -sssV_index -p434 -V.kibana -p435 -sa(dp436 -V_score -p437 -F1 -sV_type -p438 -Vvisualization -p439 -sV_id -p440 -VTop-DNS-Query -p441 -sV_source -p442 -(dp443 -VvisState -p444 -V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} -p445 -sVdescription -p446 -V -sVtitle -p447 -VTop DNS Query -p448 -sVuiStateJSON -p449 -V{} -p450 -sVversion -p451 -I1 -sVkibanaSavedObjectMeta -p452 -(dp453 -VsearchSourceJSON -p454 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p455 -sssV_index -p456 -V.kibana -p457 -sa(dp458 -V_score -p459 -F1 -sV_type -p460 -Vvisualization -p461 -sV_id -p462 -VEvent-Types -p463 -sV_source -p464 -(dp465 -VvisState -p466 -V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} -p467 -sVdescription -p468 -V -sVtitle -p469 -VEvent Sources -p470 -sVuiStateJSON -p471 -V{} -p472 -sVversion -p473 -I1 -sVkibanaSavedObjectMeta -p474 -(dp475 -VsearchSourceJSON -p476 -V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p477 -sssV_index -p478 -V.kibana -p479 -sa(dp480 -V_score -p481 -F1 -sV_type -p482 -Vvisualization -p483 -sV_id -p484 -VTotal-Events -p485 -sV_source -p486 -(dp487 -VvisState -p488 -V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}} -p489 -sVdescription -p490 -V -sVtitle -p491 -VEvent Count -p492 -sVuiStateJSON -p493 -V{} -p494 -sVversion -p495 -I1 -sVkibanaSavedObjectMeta -p496 -(dp497 -VsearchSourceJSON -p498 -V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p499 -sssV_index -p500 -V.kibana -p501 -sa(dp502 -V_score -p503 -F1 -sV_type -p504 -Vvisualization -p505 -sV_id -p506 -VUnique-Location(s) -p507 -sV_source -p508 -(dp509 -VvisState -p510 -V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}} -p511 -sVdescription -p512 -V -sVtitle -p513 -VGeo-IP Locations -p514 -sVuiStateJSON -p515 -V{} -p516 -sVversion -p517 -I1 -sVkibanaSavedObjectMeta -p518 -(dp519 -VsearchSourceJSON -p520 -V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p521 -sssV_index -p522 -V.kibana -p523 -sa(dp524 -V_score -p525 -F1 -sV_type -p526 -Vvisualization -p527 -sV_id -p528 -VTop-Alerts-By-Host -p529 -sV_source -p530 -(dp531 -VvisState -p532 -V{"title":"Top Alerts By Host","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}} -p533 -sVdescription -p534 -V -sVtitle -p535 -VTop Alerts By Host -p536 -sVuiStateJSON -p537 -V{} -p538 -sVversion -p539 -I1 -sVsavedSearchId -p540 -Vsnort-search -p541 -sVkibanaSavedObjectMeta -p542 -(dp543 -VsearchSourceJSON -p544 -V{"filter":[]} -p545 -sssV_index -p546 -V.kibana -p547 -sa(dp548 -V_score -p549 -F1 -sV_type -p550 -Vvisualization -p551 -sV_id -p552 -VTotal-Error-Messages -p553 -sV_source -p554 -(dp555 -VvisState -p556 -V{"title":"Total Errored Messages","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Total Error Messages"}}],"listeners":{}} -p557 -sVdescription -p558 -V -sVtitle -p559 -VTotal Error Messages -p560 -sVuiStateJSON -p561 -V{} -p562 -sVversion -p563 -I1 -sVkibanaSavedObjectMeta -p564 -(dp565 -VsearchSourceJSON -p566 -V{"index":"error*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p567 -sssV_index -p568 -V.kibana -p569 -sa(dp570 -V_score -p571 -F1 -sV_type -p572 -Vvisualization -p573 -sV_id -p574 -VErrors-By-Source-Type -p575 -sV_source -p576 -(dp577 -VvisState -p578 -V{\u000a "title": "Errors By Source Type",\u000a "type": "histogram",\u000a "params": {\u000a "shareYAxis": true,\u000a "addTooltip": true,\u000a "addLegend": true,\u000a "scale": "linear",\u000a "mode": "grouped",\u000a "times": [],\u000a "addTimeMarker": false,\u000a "defaultYExtents": false,\u000a "setYExtents": false,\u000a "yAxis": {}\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {\u000a "customLabel": "Count"\u000a }\u000a },\u000a {\u000a "id": "2",\u000a "type": "terms",\u000a "schema": "segment",\u000a "params": {\u000a "field": "failed_sensor_type",\u000a "size": 5,\u000a "order": "desc",\u000a "orderBy": "1"\u000a }\u000a },\u000a {\u000a "id": "4",\u000a "type": "cardinality",\u000a "schema": "metric",\u000a "params": {\u000a "field": "error_hash",\u000a "customLabel": "Unique Datapoint Count"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} -p579 -sVdescription -p580 -V -sVtitle -p581 -VErrors By Source Type -p582 -sVuiStateJSON -p583 -V{\u000a "vis": {\u000a "colors": {\u000a "Unique Datapoint Count": "#0A50A1",\u000a "Count": "#5195CE"\u000a }\u000a }\u000a} -p584 -sVversion -p585 -I1 -sVkibanaSavedObjectMeta -p586 -(dp587 -VsearchSourceJSON -p588 -V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "analyze_wildcard": true,\u000a "query": "*"\u000a }\u000a },\u000a "filter": []\u000a} -p589 -sssV_index -p590 -V.kibana -p591 -sa(dp592 -V_score -p593 -F1 -sV_type -p594 -Vvisualization -p595 -sV_id -p596 -VError-Histogram-By-Sensor-Type -p597 -sV_source -p598 -(dp599 -VvisState -p600 -V{"title":"Error Histogram By Sensor Type","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"grouped","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{},"customLabel":"Time"}},{"id":"3","type":"terms","schema":"group","params":{"field":"failed_sensor_type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p601 -sVdescription -p602 -V -sVtitle -p603 -VError Histogram By Sensor Type -p604 -sVuiStateJSON -p605 -V{} -p606 -sVversion -p607 -I1 -sVsavedSearchId -p608 -VErrors -p609 -sVkibanaSavedObjectMeta -p610 -(dp611 -VsearchSourceJSON -p612 -V{"filter":[]} -p613 -sssV_index -p614 -V.kibana -p615 -sa(dp616 -V_score -p617 -F1 -sV_type -p618 -Vdashboard -p619 -sV_id -p620 -VMetron-Dashboard -p621 -sV_source -p622 -(dp623 -Vhits -p624 -I0 -sVtimeRestore -p625 -I00 -sVdescription -p626 -V -sVtitle -p627 -VMetron Dashboard -p628 -sVuiStateJSON -p629 -V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}} -p630 -sVpanelsJSON -p631 -V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","ip_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}] -p632 -sVoptionsJSON -p633 -V{"darkTheme":false} -p634 -sVversion -p635 -I1 -sVkibanaSavedObjectMeta -p636 -(dp637 -VsearchSourceJSON -p638 -V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} -p639 -sssV_index -p640 -V.kibana -p641 -sa(dp642 -V_score -p643 -F1 -sV_type -p644 -Vindex-pattern -p645 -sV_id -p646 -Vsnort* -p647 -sV_source -p648 -(dp649 -Vfields -p650 -V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p651 -sVtimeFieldName -p652 -Vtimestamp -p653 -sVtitle -p654 -Vsnort* -p655 -ssV_index -p656 -V.kibana -p657 -sa(dp658 -V_score -p659 -F1 -sV_type -p660 -Vindex-pattern -p661 -sV_id -p662 -Vyaf* -p663 -sV_source -p664 -(dp665 -Vfields -p666 -V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p667 -sVtimeFieldName -p668 -Vtimestamp -p669 -sVtitle -p670 -Vyaf* -p671 -ssV_index -p672 -V.kibana -p673 -sa(dp674 -V_score -p675 -F1 -sV_type -p676 -Vsearch -p677 -sV_id -p678 -Vweb-search -p679 -sV_source -p680 -(dp681 -Vsort -p682 -(lp683 -Vtimestamp -p684 -aVdesc -p685 -asVhits -p686 -I0 -sVdescription -p687 -V -sVtitle -p688 -VWeb Requests -p689 -sVversion -p690 -I1 -sVkibanaSavedObjectMeta -p691 -(dp692 -VsearchSourceJSON -p693 -V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p694 -ssVcolumns -p695 -(lp696 -Vmethod -p697 -aVhost -p698 -aVuri -p699 -aVreferrer -p700 -aVip_src_addr -p701 -aVip_dst_addr -p702 -assV_index -p703 -V.kibana -p704 -sa(dp705 -V_score -p706 -F1 -sV_type -p707 -Vvisualization -p708 -sV_id -p709 -VLocation-Header -p710 -sV_source -p711 -(dp712 -VvisState -p713 -V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}} -p714 -sVdescription -p715 -V -sVtitle -p716 -VEnrichment -p717 -sVuiStateJSON -p718 -V{} -p719 -sVversion -p720 -I1 -sVkibanaSavedObjectMeta -p721 -(dp722 -VsearchSourceJSON -p723 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p724 -sssV_index -p725 -V.kibana -p726 -sa(dp727 -V_score -p728 -F1 -sV_type -p729 -Vvisualization -p730 -sV_id -p731 -VSnort-Alert-Types -p732 -sV_source -p733 -(dp734 -VvisState -p735 -V{"title":"Snort Alert Types","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"sig_id","customLabel":"Alert Type(s)"}}],"listeners":{}} -p736 -sVdescription -p737 -V -sVtitle -p738 -VSnort Alert Types -p739 -sVuiStateJSON -p740 -V{} -p741 -sVversion -p742 -I1 -sVkibanaSavedObjectMeta -p743 -(dp744 -VsearchSourceJSON -p745 -V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p746 -sssV_index -p747 -V.kibana -p748 -sa(dp749 -V_score -p750 -F1 -sV_type -p751 -Vvisualization -p752 -sV_id -p753 -VFrequent-DNS-Queries -p754 -sV_source -p755 -(dp756 -VvisState -p757 -V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p758 -sVdescription -p759 -V -sVtitle -p760 -VFrequent DNS Requests -p761 -sVuiStateJSON -p762 -V{} -p763 -sVversion -p764 -I1 -sVkibanaSavedObjectMeta -p765 -(dp766 -VsearchSourceJSON -p767 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p768 -sssV_index -p769 -V.kibana -p770 -sa(dp771 -V_score -p772 -F1 -sV_type -p773 -Vvisualization -p774 -sV_id -p775 -VDNS-Request(s) -p776 -sV_source -p777 -(dp778 -VvisState -p779 -V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p780 -sVdescription -p781 -V -sVtitle -p782 -VDNS Requests -p783 -sVuiStateJSON -p784 -V{} -p785 -sVversion -p786 -I1 -sVsavedSearchId -p787 -Vdns-search -p788 -sVkibanaSavedObjectMeta -p789 -(dp790 -VsearchSourceJSON -p791 -V{"filter":[]} -p792 -sssV_index -p793 -V.kibana -p794 -sa(dp795 -V_score -p796 -F1 -sV_type -p797 -Vvisualization -p798 -sV_id -p799 -VHTTP(S)-Requests -p800 -sV_source -p801 -(dp802 -VvisState -p803 -V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p804 -sVdescription -p805 -V -sVtitle -p806 -VWeb Requests -p807 -sVuiStateJSON -p808 -V{} -p809 -sVversion -p810 -I1 -sVsavedSearchId -p811 -Vweb-search -p812 -sVkibanaSavedObjectMeta -p813 -(dp814 -VsearchSourceJSON -p815 -V{"filter":[]} -p816 -sssV_index -p817 -V.kibana -p818 -sa(dp819 -V_score -p820 -F1 -sV_type -p821 -Vvisualization -p822 -sV_id -p823 -VErrors-Over-Time -p824 -sV_source -p825 -(dp826 -VvisState -p827 -V{\u000a "title": "Error Over Time",\u000a "type": "line",\u000a "params": {\u000a "shareYAxis": true,\u000a "addTooltip": true,\u000a "addLegend": true,\u000a "showCircles": true,\u000a "smoothLines": false,\u000a "interpolate": "linear",\u000a "scale": "linear",\u000a "drawLinesBetweenPoints": true,\u000a "radiusRatio": 9,\u000a "times": [],\u000a "addTimeMarker": true,\u000a "defaultYExtents": false,\u000a "setYExtents": false,\u000a "yAxis": {\u000a "min": 0\u000a }\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {}\u000a },\u000a {\u000a "id": "2",\u000a "type": "date_histogram",\u000a "schema": "segment",\u000a "params": {\u000a "field": "timestamp",\u000a "interval": "auto",\u000a "customInterval": "2h",\u000a "min_doc_count": 1,\u000a "extended_bounds": {}\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} -p828 -sVdescription -p829 -V -sVtitle -p830 -VErrors Over Time -p831 -sVuiStateJSON -p832 -V{} -p833 -sVversion -p834 -I1 -sVkibanaSavedObjectMeta -p835 -(dp836 -VsearchSourceJSON -p837 -V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "query": "*",\u000a "analyze_wildcard": true\u000a }\u000a },\u000a "filter": []\u000a} -p838 -sssV_index -p839 -V.kibana -p840 -sa(dp841 -V_score -p842 -F1 -sV_type -p843 -Vvisualization -p844 -sV_id -p845 -VError-Source-Proportion -p846 -sV_source -p847 -(dp848 -VvisState -p849 -V{\u000a "title": "Sensor Type Proportion",\u000a "type": "pie",\u000a "params": {\u000a "shareYAxis": true,\u000a "addTooltip": true,\u000a "addLegend": true,\u000a "isDonut": false\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {}\u000a },\u000a {\u000a "id": "2",\u000a "type": "terms",\u000a "schema": "segment",\u000a "params": {\u000a "field": "failed_sensor_type",\u000a "size": 5,\u000a "order": "desc",\u000a "orderBy": "1",\u000a "customLabel": "Sensor"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} -p850 -sVdescription -p851 -V -sVtitle -p852 -VError Source Proportion -p853 -sVuiStateJSON -p854 -V{} -p855 -sVversion -p856 -I1 -sVkibanaSavedObjectMeta -p857 -(dp858 -VsearchSourceJSON -p859 -V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "query": "*",\u000a "analyze_wildcard": true\u000a }\u000a },\u000a "filter": []\u000a} -p860 -sssV_index -p861 -V.kibana -p862 -sa(dp863 -V_score -p864 -F1 -sV_type -p865 -Vindex-pattern -p866 -sV_id -p867 -Verror* -p868 -sV_source -p869 -(dp870 -Vfields -p871 -V[{"name":"exception","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"stack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"error_hash","type":"string","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"raw_message","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"message","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"failed_sensor_type","type":"string","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"hostname","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":1,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"error_type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"error_fields","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"raw_message_bytes","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"timestamp","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p872 -sVtimeFieldName -p873 -Vtimestamp -p874 -sVtitle -p875 -Verror* -p876 -ssV_index -p877 -V.kibana -p878 -sa(dp879 -V_score -p880 -F1 -sV_type -p881 -Vvisualization -p882 -sV_id -p883 -VError-Date-Histogram -p884 -sV_source -p885 -(dp886 -VvisState -p887 -V{"title":"New Visualization","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{},"customLabel":"Time"}}],"listeners":{}} -p888 -sVdescription -p889 -V -sVtitle -p890 -VError Date Histogram -p891 -sVuiStateJSON -p892 -V{} -p893 -sVversion -p894 -I1 -sVsavedSearchId -p895 -VErrors -p896 -sVkibanaSavedObjectMeta -p897 -(dp898 -VsearchSourceJSON -p899 -V{"filter":[]} -p900 -sssV_index -p901 -V.kibana -p902 -sa(dp903 -V_score -p904 -F1 -sV_type -p905 -Vdashboard -p906 -sV_id -p907 -VMetron-Error-Dashboard -p908 -sV_source -p909 -(dp910 -Vhits -p911 -I0 -sVtimeRestore -p912 -I00 -sVdescription -p913 -V -sVtitle -p914 -VMetron Error Dashboard -p915 -sVuiStateJSON -p916 -V{"P-2":{"vis":{"legendOpen":true}},"P-23":{"vis":{"colors":{"amb3.service.consul":"#629E51","host":"#629E51","host2":"#9AC48A","hostAnother":"#7EB26D","hostNew":"#B7DBAB"}}},"P-3":{"vis":{"colors":{"fourth":"#1F78C1","new_error":"#BADFF4","test_error":"#82B5D8"}}},"P-5":{"vis":{"colors":{"another_new_parser_error":"#806EB7","new_parser_error":"#AEA2E0","parser_error":"#614D93"}}}} -p917 -sVpanelsJSON -p918 -V[{"col":5,"id":"Errors-By-Error-Type","panelIndex":2,"row":9,"size_x":8,"size_y":3,"type":"visualization"},{"col":1,"id":"Error-Source-Proportion","panelIndex":3,"row":9,"size_x":4,"size_y":3,"type":"visualization"},{"col":5,"id":"Errors-By-Source-Type","panelIndex":4,"row":12,"size_x":8,"size_y":3,"type":"visualization"},{"col":1,"id":"Error-Type-Proportion","panelIndex":5,"row":12,"size_x":4,"size_y":3,"type":"visualization"},{"col":8,"id":"Unique-Error-Messages","panelIndex":19,"row":1,"size_x":4,"size_y":2,"type":"visualization"},{"col":3,"id":"Total-Error-Messages","panelIndex":20,"row":1,"size_x":4,"size_y":2,"type":"visualization"},{"col":5,"id":"Errors-By-Hostname","panelIndex":22,"row":15,"size_x":8,"size_y":3,"type":"visualization"},{"col":1,"id":"Error-Hostname-Proportion","panelIndex":23,"row":15,"size_x":4,"size_y":3,"type":"visualization"},{"col":1,"columns":["failed_sensor_type","error_type","exception","hostname","message","raw_message","error_hash"],"id":"Errors","panelIndex":25,"row":18,"size_x":12,"size_y":7,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Error-Histogram-By-Sensor-Type","panelIndex":27,"row":3,"size_x":12,"size_y":3,"type":"visualization"},{"id":"Unique-Error-Histogram-By-Sensor-Type","type":"visualization","panelIndex":28,"size_x":12,"size_y":3,"col":1,"row":6}] -p919 -sVoptionsJSON -p920 -V{"darkTheme":false} -p921 -sVversion -p922 -I1 -sVkibanaSavedObjectMeta -p923 -(dp924 -VsearchSourceJSON -p925 -V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} -p926 -sssV_index -p927 -V.kibana -p928 -sa(dp929 -V_score -p930 -F1 -sV_type -p931 -Vconfig -p932 -sV_id -p933 -V4.5.3 -p934 -sV_source -p935 -(dp936 -VbuildNum -p937 -I9892 -sVdefaultIndex -p938 -Vbro* -p939 -ssV_index -p940 -V.kibana -p941 -sa(dp942 -V_score -p943 -F1 -sV_type -p944 -Vsearch -p945 -sV_id -p946 -Vdns-search -p947 -sV_source -p948 -(dp949 -Vsort -p950 -(lp951 -Vtimestamp -p952 -aVdesc -p953 -asVhits -p954 -I0 -sVdescription -p955 -V -sVtitle -p956 -VDNS Requests -p957 -sVversion -p958 -I1 -sVkibanaSavedObjectMeta -p959 -(dp960 -VsearchSourceJSON -p961 -V{"index":"bro*","query":{"query_string":{"query":"protocol: dns","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p962 -ssVcolumns -p963 -(lp964 -Vquery -p965 -aVqtype_name -p966 -aVanswers -p967 -aVip_src_addr -p968 -aVip_dst_addr -p969 -assV_index -p970 -V.kibana -p971 -sa(dp972 -V_score -p973 -F1 -sV_type -p974 -Vvisualization -p975 -sV_id -p976 -VDNS-Requests-Header -p977 -sV_source -p978 -(dp979 -VvisState -p980 -V{"aggs":[],"listeners":{},"params":{"markdown":"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network."},"title":"DNS Requests","type":"markdown"} -p981 -sVdescription -p982 -V -sVtitle -p983 -VDNS Requests -p984 -sVuiStateJSON -p985 -V{} -p986 -sVversion -p987 -I1 -sVkibanaSavedObjectMeta -p988 -(dp989 -VsearchSourceJSON -p990 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p991 -sssV_index -p992 -V.kibana -p993 -sa(dp994 -V_score -p995 -F1 -sV_type -p996 -Vvisualization -p997 -sV_id -p998 -VYAF-Flows-Header -p999 -sV_source -p1000 -(dp1001 -VvisState -p1002 -V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}} -p1003 -sVdescription -p1004 -V -sVtitle -p1005 -VYAF -p1006 -sVuiStateJSON -p1007 -V{} -p1008 -sVversion -p1009 -I1 -sVkibanaSavedObjectMeta -p1010 -(dp1011 -VsearchSourceJSON -p1012 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p1013 -sssV_index -p1014 -V.kibana -p1015 -sa(dp1016 -V_score -p1017 -F1 -sV_type -p1018 -Vvisualization -p1019 -sV_id -p1020 -VTop-5-Exceptions -p1021 -sV_source -p1022 -(dp1023 -VvisState -p1024 -V{"title":"Top-5 Exceptions","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"exception","size":5,"order":"desc","orderBy":"1","customLabel":"Exceptions"}}],"listeners":{}} -p1025 -sVdescription -p1026 -V -sVtitle -p1027 -VTop-5 Exceptions -p1028 -sVuiStateJSON -p1029 -V{} -p1030 -sVversion -p1031 -I1 -sVkibanaSavedObjectMeta -p1032 -(dp1033 -VsearchSourceJSON -p1034 -V{"index":"error*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p1035 -sssV_index -p1036 -V.kibana -p1037 -sa(dp1038 -V_score -p1039 -F1 -sV_type -p1040 -Vvisualization -p1041 -sV_id -p1042 -VFrequent-DNS-Requests -p1043 -sV_source -p1044 -(dp1045 -VvisState -p1046 -V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1","customLabel":"DNS Query"}}],"listeners":{}} -p1047 -sVdescription -p1048 -V -sVtitle -p1049 -VFrequent DNS Requests -p1050 -sVuiStateJSON -p1051 -V{} -p1052 -sVversion -p1053 -I1 -sVkibanaSavedObjectMeta -p1054 -(dp1055 -VsearchSourceJSON -p1056 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p1057 -sssV_index -p1058 -V.kibana -p1059 -sa(dp1060 -V_score -p1061 -F1 -sV_type -p1062 -Vvisualization -p1063 -sV_id -p1064 -VCountry -p1065 -sV_source -p1066 -(dp1067 -VvisState -p1068 -V{"title":"By Country","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"enrichments:geo:ip_src_addr:country","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p1069 -sVdescription -p1070 -V -sVtitle -p1071 -VBy Country -p1072 -sVuiStateJSON -p1073 -V{} -p1074 -sVversion -p1075 -I1 -sVkibanaSavedObjectMeta -p1076 -(dp1077 -VsearchSourceJSON -p1078 -V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p1079 -sssV_index -p1080 -V.kibana -p1081 -sa(dp1082 -V_score -p1083 -F1 -sV_type -p1084 -Vvisualization -p1085 -sV_id -p1086 -VTop-Destinations -p1087 -sV_source -p1088 -(dp1089 -VvisState -p1090 -V{"title":"Top Destinations","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Destination IP"}}],"listeners":{}} -p1091 -sVdescription -p1092 -V -sVtitle -p1093 -VTop Destinations -p1094 -sVuiStateJSON -p1095 -V{} -p1096 -sVversion -p1097 -I1 -sVkibanaSavedObjectMeta -p1098 -(dp1099 -VsearchSourceJSON -p1100 -V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p1101 -sssV_index -p1102 -V.kibana -p1103 -sa(dp1104 -V_score -p1105 -F1 -sV_type -p1106 -Vvisualization -p1107 -sV_id -p1108 -VUnusual-Referrers -p1109 -sV_source -p1110 -(dp1111 -VvisState -p1112 -V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}} -p1113 -sVdescription -p1114 -V -sVtitle -p1115 -VUnusual Referrers -p1116 -sVuiStateJSON -p1117 -V{} -p1118 -sVversion -p1119 -I1 -sVsavedSearchId -p1120 -Vweb-search -p1121 -sVkibanaSavedObjectMeta -p1122 -(dp1123 -VsearchSourceJSON -p1124 -V{"filter":[]} -p1125 -sssV_index -p1126 -V.kibana -p1127 -sa(dp1128 -V_score -p1129 -F1 -sV_type -p1130 -Vvisualization -p1131 -sV_id -p1132 -VUnique-Error-Histogram-By-Sensor-Type -p1133 -sV_source -p1134 -(dp1135 -VvisState -p1136 -V{"title":"Error Histogram By Sensor Type","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"grouped","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"error_hash"}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{},"customLabel":"Time"}},{"id":"3","type":"terms","schema":"group","params":{"field":"failed_sensor_type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p1137 -sVdescription -p1138 -V -sVtitle -p1139 -VUnique Error Histogram By Sensor Type -p1140 -sVuiStateJSON -p1141 -V{} -p1142 -sVversion -p1143 -I1 -sVsavedSearchId -p1144 -VErrors -p1145 -sVkibanaSavedObjectMeta -p1146 -(dp1147 -VsearchSourceJSON -p1148 -V{"filter":[]} -p1149 -sssV_index -p1150 -V.kibana -p1151 -sa(dp1152 -V_score -p1153 -F1 -sV_type -p1154 -Vvisualization -p1155 -sV_id -p1156 -VFlow-Locations -p1157 -sV_source -p1158 -(dp1159 -VvisState -p1160 -V{"title":"Flow Locations","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":true,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}} -p1161 -sVdescription -p1162 -V -sVtitle -p1163 -VFlow Locations -p1164 -sVuiStateJSON -p1165 -V{} -p1166 -sVversion -p1167 -I1 -sVkibanaSavedObjectMeta -p1168 -(dp1169 -VsearchSourceJSON -p1170 -V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p1171 -sssV_index -p1172 -V.kibana -p1173 -sa. \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py index 87d678886c..dfdb724f37 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py @@ -24,6 +24,7 @@ from ambari_commons.os_check import OSCheck from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ansible.modules.extras.packaging import kibana_plugin from resource_management.core.logger import Logger from resource_management.core.resources.system import Directory from resource_management.core.resources.system import Execute @@ -98,8 +99,6 @@ def status(self, env): @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def load_template(self, env): - from dashboard.dashboardindex import DashboardIndex - import params env.set_params(params) @@ -107,21 +106,17 @@ def load_template(self, env): port = int(ambari_format("{es_port}")) Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port)) - di = DashboardIndex(host=hostname, port=port) - - # Loads Kibana Dashboard definition from disk and replaces .kibana on index - templateFile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard.p') - if not os.path.isfile(templateFile): - raise IOError( - errno.ENOENT, os.strerror(errno.ENOENT), templateFile) - - Logger.info("Deleting .kibana index from Elasticsearch") - di.es.indices.delete(index='.kibana', ignore=[400, 404]) + kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json') + if not os.path.isfile(kibanaDashboardLoad): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad) - Logger.info("Loading .kibana index from %s" % templateFile) + Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad) - di.put(data=di.load(filespec=templateFile)) + kibana_cmd = ambari_format( + 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_host}:{es_port}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad) + Execute(kibana_cmd, logoutput=True) if __name__ == "__main__": diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py index a5e0ec96c3..ced626ab49 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py @@ -31,7 +31,7 @@ kibana_home = '/usr/share/kibana/' kibana_bin = '/usr/share/kibana/bin/' -conf_dir = "/opt/kibana/config/" +conf_dir = "/etc/kibana" kibana_user = config['configurations']['kibana-env']['kibana_user'] kibana_group = config['configurations']['kibana-env']['kibana_group'] log_dir = config['configurations']['kibana-env']['kibana_log_dir'] diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java new file mode 100644 index 0000000000..917a02f924 --- /dev/null +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.elasticsearch.bulk; + +import com.fasterxml.jackson.core.type.TypeReference; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.metron.common.utils.JSONUtils; + +public class ElasticsearchImportExport { + + public static void main(String[] args) { + if (args.length != 2) { + throw new RuntimeException("Expects 'input' and 'output' file arguments."); + } + final String inPath = args[0]; + final String outPath = args[1]; + try { + new ElasticsearchImportExport().bulkify(Paths.get(inPath), Paths.get(outPath)); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + System.exit(0); + } + + public void bulkify(Path input, Path output) throws IOException { + List outRecs = new ArrayList(); + try (BufferedReader br = new BufferedReader(new FileReader(input.toFile()))) { + String line; + while ((line = br.readLine()) != null) { + Map inDoc = JSONUtils.INSTANCE + .load(line, new TypeReference>() { + }); + Object id = inDoc.get("_id"); + Object type = inDoc.get("_type"); + String createRaw = String + .format("{ \"create\" : { \"_id\": \"%s\", \"_type\": \"%s\" } }", id, type); + String outData = JSONUtils.INSTANCE.toJSON(inDoc.get("_source"), false); + outRecs.add(createRaw); + outRecs.add(outData); + } + } + try (BufferedWriter br = new BufferedWriter(new FileWriter(output.toFile()))) { + for (String line : outRecs) { + br.write(line); + br.write(System.lineSeparator()); + } + } + } + +} diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java new file mode 100644 index 0000000000..77315f9a65 --- /dev/null +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.elasticsearch.bulk; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.integration.utils.TestUtils; +import org.junit.Before; +import org.junit.Test; + +public class ElasticsearchImportExportTest { + + + /** + *{"_index":".kibana","_type":"visualization","_id":"AV-Sj0e2hKs1cXXnFMqF","_score":1,"_source":{"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}} + *{"_index":".kibana","_type":"blah","_id":"MIKE-AV-Sj0e2hKs1cXXnFMqF","_score":1,"_source":{"title":"another Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}} + */ + @Multiline + private static String records; + + /** + *{ "create" : { "_id": "AV-Sj0e2hKs1cXXnFMqF", "_type": "visualization" } } + *{"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} + *{ "create" : { "_id": "MIKE-AV-Sj0e2hKs1cXXnFMqF", "_type": "blah" } } + *{"title":"another Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} + */ + @Multiline + private static String expected; + private File tempDir; + + @Before + public void setup() throws Exception { + tempDir = TestUtils.createTempDir(this.getClass().getName()); + } + + @Test + public void bulk_exporter_writes_elasticsearch_records_in_bulk_import_format() throws Exception { + Path recordsFile = Paths.get(tempDir.getPath(), "inputfile.json"); + Path outputFile = Paths.get(tempDir.getPath(), "outputfile.json"); + TestUtils.write(recordsFile.toFile(), records); + + ElasticsearchImportExport tool = new ElasticsearchImportExport(); + tool.bulkify(recordsFile, outputFile); + String actual = TestUtils.read(outputFile.toFile()); + assertThat(actual, equalTo(expected)); + } + +} diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java index 9577a43500..0c37a35d35 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java @@ -21,10 +21,12 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; @@ -32,6 +34,7 @@ public class TestUtils { public static long MAX_ASSERT_WAIT_MS = 30000L; + public interface Assertion { void apply() throws Exception; } @@ -87,6 +90,30 @@ public static File write(File file, String contents) throws IOException { return file; } + /** + * Reads file contents into a String. Uses UTF-8 as default charset. + * + * @param in Input file + * @return contents of input file + * @throws IOException + */ + public static String read(File in) throws IOException { + return read(in, StandardCharsets.UTF_8); + } + + /** + * Reads file contents into a String + * + * @param in Input file + * @param charset charset to use for reading + * @return contents of input file + * @throws IOException + */ + public static String read(File in, Charset charset) throws IOException { + byte[] bytes = Files.readAllBytes(Paths.get(in.getPath())); + return new String(bytes, charset); + } + /** * Cleans up after test run via runtime shutdown hooks */ diff --git a/pom.xml b/pom.xml index 242d8632e1..445cd966ba 100644 --- a/pom.xml +++ b/pom.xml @@ -327,8 +327,6 @@ **/dependency-reduced-pom.xml **/target/** **/bro-plugin-kafka/build/** - - **/src/main/resources/common-services/KIBANA/**/package/scripts/dashboard/dashboard.p **/packer-build/scripts/** **/packer-build/bin/** From 364452624f63c6cd1f4e435e560b2603619eddde Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 10 Nov 2017 09:55:17 -0700 Subject: [PATCH 34/59] Address Otto's PR comments. Fix Kibana in the Ambari Blueprint. --- .../5.6.2/package/scripts/slave.py | 2 +- .../5.6.2/package/scripts/kibana_master.py | 2 -- .../ambari_config/vars/single_node_vm.yml | 1 + .../ambari_config/vars/small_cluster.yml | 1 + metron-platform/elasticsearch-shaded/pom.xml | 1 + .../metron-data-management/pom.xml | 29 ++----------------- .../bulk/ElasticsearchImportExport.java | 17 +++++++++++ .../bulk/ElasticsearchImportExportTest.java | 8 ++--- 8 files changed, 27 insertions(+), 34 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py index 3303b18c62..a19989eddc 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py @@ -53,7 +53,7 @@ def slave(): elastic_site = params.config['configurations']['elastic-site'] path = "{0}/elasticsearch.yml".format(params.conf_dir) - Logger.info("Cre") + Logger.info("Creating ES slave configuration.") File(path, content=Template( "elasticsearch.slave.yaml.j2", diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py index dfdb724f37..731dd26b5a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py @@ -22,9 +22,7 @@ import errno import os -from ambari_commons.os_check import OSCheck from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl -from ansible.modules.extras.packaging import kibana_plugin from resource_management.core.logger import Logger from resource_management.core.resources.system import Directory from resource_management.core.resources.system import Execute diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml index 024582cb13..a818ecd50c 100644 --- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml @@ -121,6 +121,7 @@ required_configurations: kibana_log_dir: /var/log/kibana kibana_server_port: 5000 kibana_default_application: "dashboard/Metron-Dashboard" + kibana_server_host: 0.0.0.0 blueprint: stack_name: HDP diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml index 6bdc0b4c0e..0347078e21 100644 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml @@ -113,6 +113,7 @@ required_configurations: kibana_log_dir: /var/log/kibana kibana_server_port: 5000 kibana_default_application: "dashboard/Metron-Dashboard" + kibana_server_host: 0.0.0.0 blueprint: stack_name: HDP diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index 2addc9f5d0..744602de3b 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -149,6 +149,7 @@ + io.netty org.apache.metron.io.netty diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml index 1b0ed99ac8..42ae53df3e 100644 --- a/metron-platform/metron-data-management/pom.xml +++ b/metron-platform/metron-data-management/pom.xml @@ -29,6 +29,7 @@ 4.3.2 5.5.0 + com.google.guava @@ -144,7 +145,6 @@ ${global_hadoop_version} provided - org.apache.hadoop hadoop-auth @@ -189,7 +189,6 @@ provided - org.apache.hbase hbase-server @@ -207,8 +206,6 @@ org.apache.hadoop hadoop-hdfs - - org.apache.hadoop hadoop-common @@ -231,11 +228,6 @@ httpclient ${httpcore.version} - org.hamcrest hamcrest-all @@ -248,24 +240,6 @@ 2.1.14 test - org.apache.hadoop hadoop-hdfs @@ -371,6 +345,7 @@ test + diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java index 917a02f924..0a04dfc724 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java @@ -31,6 +31,14 @@ import java.util.Map; import org.apache.metron.common.utils.JSONUtils; +/** + * This is a utility for taking a file of JSON objects that were exported from ES and transforming + * it into a bulk import format. This was useful for backing up and restoring the Kibana dashboard + * index. The notable gap is that it expects one record per line in the file, which is not how + * ES generally returns results. Elasticsearch-dump was used as the intermediary to export data in + * the desired format for consumption by this tool. + * @see https://github.com/taskrabbit/elasticsearch-dump + */ public class ElasticsearchImportExport { public static void main(String[] args) { @@ -48,6 +56,15 @@ public static void main(String[] args) { System.exit(0); } + /** + * Takes a file of line-delimited JSON objects and converts them to an Elasticsearch bulk import + * format. + * + * @param input input JSON file (note, each line is expected to be a separate complete JSON + * object, not the file as a whole.) + * @param output Elasticsearch bulk import file. + * @throws IOException + */ public void bulkify(Path input, Path output) throws IOException { List outRecs = new ArrayList(); try (BufferedReader br = new BufferedReader(new FileReader(input.toFile()))) { diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java index 77315f9a65..ddec27cda0 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java @@ -33,17 +33,17 @@ public class ElasticsearchImportExportTest { /** - *{"_index":".kibana","_type":"visualization","_id":"AV-Sj0e2hKs1cXXnFMqF","_score":1,"_source":{"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}} - *{"_index":".kibana","_type":"blah","_id":"MIKE-AV-Sj0e2hKs1cXXnFMqF","_score":1,"_source":{"title":"another Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}} + *{"_index":".kibana","_type":"visualization","_id":"AV-Sj0e2hKs1cXXnFMqF","_score":1,"_source":{"title":"Welcome to Apache Metron"}} + *{"_index":".kibana","_type":"blah","_id":"MIKE-AV-Sj0e2hKs1cXXnFMqF","_score":1,"_source":{"title":"another Welcome to Apache Metron"}} */ @Multiline private static String records; /** *{ "create" : { "_id": "AV-Sj0e2hKs1cXXnFMqF", "_type": "visualization" } } - *{"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} + *{"title":"Welcome to Apache Metron"} *{ "create" : { "_id": "MIKE-AV-Sj0e2hKs1cXXnFMqF", "_type": "blah" } } - *{"title":"another Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} + *{"title":"another Welcome to Apache Metron"} */ @Multiline private static String expected; From 7d41ad24ea154bf0c8a92bb44639f4bebf31bc78 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Fri, 10 Nov 2017 16:38:33 -0700 Subject: [PATCH 35/59] Fix remaining Kibana drama --- .../manual-install/Manual_Install_CentOS6.md | 2 +- .../5.6.2/configuration/elastic-sysconfig.xml | 2 +- .../package/scripts/dashboard/kibana.template | 233 ++++++++++++++++++ .../5.6.2/package/scripts/kibana_master.py | 10 + .../KIBANA/5.6.2/package/scripts/params.py | 1 + .../ambari_config/vars/single_node_vm.yml | 2 +- .../ambari_config/vars/small_cluster.yml | 2 +- .../elasticsearch/dao/ElasticsearchDao.java | 33 ++- 8 files changed, 263 insertions(+), 22 deletions(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template diff --git a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md index 052086d3e0..8fe106ef02 100644 --- a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md +++ b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md @@ -441,7 +441,7 @@ Client - Kibana: * Set "kibana_es_url" to `http://:9200`. "replace_with_elasticsearch_master_hostname" is the IP of the node where you assigned ElasticSearch Master on the Assign Master tab. - * Change kibana_default_application to "dashboard/Metron-Dashboard" + * Change kibana_default_application to "dashboard/AV-YpDmwdXwc6Ua9Muh9" - Metron: Set "Elasticsearch Hosts" to the IP of the node where you assigned ElasticSearch Master on the Assign Master tab. diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml index 6779a1765c..d6db027e6d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml @@ -41,7 +41,7 @@ heap_size - 128m + 512m Heap size diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template new file mode 100644 index 0000000000..6f38ed5b98 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template @@ -0,0 +1,233 @@ +{ + "template" : ".kibana", + "mappings" : { + "search" : { + "dynamic" : "strict", + "properties" : { + "columns" : { + "type" : "keyword" + }, + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "sort" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "url" : { + "dynamic" : "strict", + "properties" : { + "accessCount" : { + "type" : "long" + }, + "accessDate" : { + "type" : "date" + }, + "createDate" : { + "type" : "date" + }, + "url" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 2048 + } + } + } + } + }, + "dashboard" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "optionsJSON" : { + "type" : "text" + }, + "panelsJSON" : { + "type" : "text" + }, + "refreshInterval" : { + "properties" : { + "display" : { + "type" : "keyword" + }, + "pause" : { + "type" : "boolean" + }, + "section" : { + "type" : "integer" + }, + "value" : { + "type" : "integer" + } + } + }, + "timeFrom" : { + "type" : "keyword" + }, + "timeRestore" : { + "type" : "boolean" + }, + "timeTo" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "uiStateJSON" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "index-pattern" : { + "dynamic" : "strict", + "properties" : { + "fieldFormatMap" : { + "type" : "text" + }, + "fields" : { + "type" : "text" + }, + "intervalName" : { + "type" : "keyword" + }, + "notExpandable" : { + "type" : "boolean" + }, + "sourceFilters" : { + "type" : "text" + }, + "timeFieldName" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + } + } + }, + "timelion-sheet" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "timelion_chart_height" : { + "type" : "integer" + }, + "timelion_columns" : { + "type" : "integer" + }, + "timelion_interval" : { + "type" : "keyword" + }, + "timelion_other_interval" : { + "type" : "keyword" + }, + "timelion_rows" : { + "type" : "integer" + }, + "timelion_sheet" : { + "type" : "text" + }, + "title" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "visualization" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "savedSearchId" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "uiStateJSON" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + }, + "visState" : { + "type" : "text" + } + } + }, + "server" : { + "dynamic" : "strict", + "properties" : { + "uuid" : { + "type" : "keyword" + } + } + }, + "_default_" : { + "dynamic" : "strict" + }, + "config" : { + "dynamic" : "true", + "properties" : { + "buildNum" : { + "type" : "keyword" + } + } + } + } +} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py index 731dd26b5a..4c001efece 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py @@ -105,6 +105,16 @@ def load_template(self, env): Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port)) + kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template') + if not os.path.isfile(kibanaTemplate): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate) + + Logger.info("Loading .kibana index template from %s" % kibanaTemplate) + template_cmd = ambari_format( + 'curl -s -XPOST http://{es_host}:{es_port}/_template/.kibana -d @%s' % kibanaTemplate) + Execute(template_cmd, logoutput=True) + kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json') if not os.path.isfile(kibanaDashboardLoad): raise IOError( diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py index ced626ab49..ef4cb622c6 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py @@ -42,6 +42,7 @@ es_host = parsed.netloc.split(':')[0] es_port = parsed.netloc.split(':')[1] kibana_port = config['configurations']['kibana-env']['kibana_server_port'] +kibana_server_host = config['configurations']['kibana-env']['kibana_server_host'] kibana_default_application = config['configurations']['kibana-env']['kibana_default_application'] hostname = config['hostname'] java64_home = config['hostLevelParams']['java_home'] diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml index a818ecd50c..839e04db22 100644 --- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml @@ -120,7 +120,7 @@ required_configurations: kibana_es_url: http://{{ groups.search[0] }}:9200 kibana_log_dir: /var/log/kibana kibana_server_port: 5000 - kibana_default_application: "dashboard/Metron-Dashboard" + kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9" kibana_server_host: 0.0.0.0 blueprint: diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml index 0347078e21..8bfb9e8fab 100644 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml @@ -112,7 +112,7 @@ required_configurations: kibana_es_url: http://{{ groups.web[0] }}:9200 kibana_log_dir: /var/log/kibana kibana_server_port: 5000 - kibana_default_application: "dashboard/Metron-Dashboard" + kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9" kibana_server_host: 0.0.0.0 blueprint: diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 0da87688ce..eac6b0a36c 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -17,8 +17,23 @@ */ package org.apache.metron.elasticsearch.dao; +import static org.apache.metron.elasticsearch.utils.ElasticsearchUtils.INDEX_NAME_DELIMITER; + import com.google.common.base.Splitter; import com.google.common.collect.Iterables; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; @@ -64,22 +79,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.apache.metron.elasticsearch.utils.ElasticsearchUtils.INDEX_NAME_DELIMITER; - public class ElasticsearchDao implements IndexDao { @@ -141,8 +140,6 @@ protected SearchResponse search(SearchRequest searchRequest, QueryBuilder queryB searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); Optional> fields = searchRequest.getFields(); if (fields.isPresent()) { -// searchSourceBuilder.storedFields(fields.get()); -// searchSourceBuilder.fetchSource(fields.get().toArray(new String[]{}), null); searchSourceBuilder.fetchSource("*", null); } else { searchSourceBuilder.fetchSource(true); From d31418b40046f415ed8cdfc157535f6d448d7962 Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 14 Nov 2017 15:31:11 -0700 Subject: [PATCH 36/59] Remove commented out lines --- .../integration/ElasticsearchSearchIntegrationTest.java | 4 ---- pom.xml | 1 - 2 files changed, 5 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 4a619e9704..3e36c7c936 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -191,10 +191,6 @@ protected void loadTestData() if (bulkResponse.hasFailures()) { throw new RuntimeException("Failed to index test data"); } -// SearchRequestBuilder metaalerts = es.getClient().prepareSearch("metaalerts") -// .setQuery(QueryBuilders.matchAllQuery()).setFetchSource(true); -// SearchResponse response = metaalerts.get(); -// System.out.println("blah"); SearchResponse broDocs = es.getClient() .prepareSearch("bro_index_2017.01.01.01") diff --git a/pom.xml b/pom.xml index 445cd966ba..caf5748b8a 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,6 @@ ${base_hbase_version} ${base_flume_version} 5.6.2 - 1.1.1 3.0.2 4.12 From 94e692cce8e3da802f441e2daf35c8f6dc71418e Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 15 Nov 2017 08:38:53 -0700 Subject: [PATCH 37/59] Fix string-to-keyword ES mapping --- .../METRON/CURRENT/package/files/bro_index.template | 4 ++-- .../METRON/CURRENT/package/files/meta_index.template | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template index 84ab1703bf..69014d2075 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template @@ -125,7 +125,7 @@ * Be careful when modifying this file to not unintentionally affect other logs. * For instance, the "version" field exists in the HTTP, SSL, and SSH logs. If you * were to only consider the SSH log, you would set the type to integer, but because - * in the SSL and HTTP logs version is a string, we must set the type to string. + * in the SSL and HTTP logs version is a string, we must set the type to keyword. */ /* * Metron-specific fields @@ -809,7 +809,7 @@ * Notes: In other bro records, the id field is of type conn_id, so it is * expanded before being logged into 4 fields, all of which are addressed * under the "Widely-used Bro fields" section of this template. In X509 - * logs, however, id is a string to identify the certificate file id. + * logs, however, id is a keyword to identify the certificate file id. */ "id": { "type": "keyword" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template index 258056d732..5f77a09b2a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template @@ -31,8 +31,7 @@ "type": "nested" }, "source:type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } } From 11e41f27d31c76d25f68aa92ae0d5eff347e96da Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 16 Nov 2017 12:04:10 -0500 Subject: [PATCH 38/59] Test fixes. --- metron-platform/metron-elasticsearch/pom.xml | 4 +- .../elasticsearch/dao/ElasticsearchDao.java | 15 +++-- ...ElasticsearchMetaAlertIntegrationTest.java | 67 ++++++++++--------- .../indexing/dao/SearchIntegrationTest.java | 29 ++++---- 4 files changed, 59 insertions(+), 56 deletions(-) diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 8cb43f06ef..3699be9679 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -33,12 +33,12 @@ elasticsearch-shaded ${project.parent.version} - + org.apache.metron metron-enrichment diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index cf8c590d84..f99a3edd3e 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -65,6 +65,7 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.mapper.LegacyIpFieldMapper; +import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; @@ -311,16 +312,18 @@ List searchByGuids(Collection guids, Collection sensorTyp return Collections.EMPTY_LIST; } QueryBuilder query = null; + IdsQueryBuilder idsQuery = null; if (sensorTypes != null) { String[] types = sensorTypes.stream().map(sensorType -> sensorType + "_doc").toArray(String[]::new); - for(String guid : guids) { - query = QueryBuilders.idsQuery(types).addIds(guid); - } + idsQuery = QueryBuilders.idsQuery(types); } else { - for(String guid : guids) { - query = QueryBuilders.idsQuery().addIds(guid); - } + idsQuery = QueryBuilders.idsQuery(); } + + for(String guid : guids) { + query = idsQuery.addIds(guid); + } + SearchRequestBuilder request = client.prepareSearch() .setQuery(query) .setSize(guids.size()) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index c28094beba..416a516ea0 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -29,16 +29,11 @@ import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; + +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; @@ -160,6 +155,29 @@ public class ElasticsearchMetaAlertIntegrationTest { @Multiline public static String statusPatchRequest; + /** + * { + "metaalert_doc" : { + "properties" : { + "guid" : { + "type" : "keyword" + }, + "ip_src_addr" : { + "type" : "keyword" + }, + "score" : { + "type" : "integer" + }, + "alert" : { + "type" : "nested" + } + } + } + } + */ + @Multiline + public static String templates; + @BeforeClass public static void setupBefore() throws Exception { // setup the client @@ -190,7 +208,7 @@ public static void setupBefore() throws Exception { @Before public void setup() throws IOException { es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, - buildMetaMappingSource()); + templates); } @AfterClass @@ -205,27 +223,6 @@ public void reset() { es.reset(); } - protected static String buildMetaMappingSource() throws IOException { - return jsonBuilder().prettyPrint() - .startObject() - .startObject(MetaAlertDao.METAALERT_DOC) - .startObject("properties") - .startObject("guid") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .startObject("score") - .field("type", "integer") - .field("index", "not_analyzed") - .endObject() - .startObject("alert") - .field("type", "nested") - .endObject() - .endObject() - .endObject() - .endObject() - .string(); - } @Test public void shouldGetAllMetaAlertsForAlert() throws Exception { @@ -260,7 +257,11 @@ public void shouldGetAllMetaAlertsForAlert() throws Exception { SearchResponse searchResponse0 = metaDao.getAllMetaAlertsForAlert("message_0"); List searchResults0 = searchResponse0.getResults(); Assert.assertEquals(13, searchResults0.size()); - Assert.assertEquals(metaAlerts.get(0), searchResults0.get(0).getSource()); + Set> resultSet = new HashSet<>(); + Iterables.addAll(resultSet, Iterables.transform(searchResults0, r -> r.getSource())); + StringBuffer reason = new StringBuffer("Unable to find " + metaAlerts.get(0) + "\n"); + reason.append(Joiner.on("\n").join(resultSet)); + Assert.assertTrue(reason.toString(), resultSet.contains(metaAlerts.get(0))); // Verify no meta alerts are returned because message_1 was not added to any SearchResponse searchResponse1 = metaDao.getAllMetaAlertsForAlert("message_1"); @@ -744,7 +745,7 @@ public void shouldSearchByNestedAlert() throws Exception { + " OR (alert.ip_src_addr:192.168.1.3 AND alert.ip_src_port:8008)"); setIndices(Collections.singletonList("*")); setFrom(0); - setSize(5); + setSize(1); setSort(Collections.singletonList(new SortField() { { setField(Constants.GUID); diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index a254e2195b..d1a58741a1 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -17,14 +17,9 @@ */ package org.apache.metron.indexing.dao; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; + import com.fasterxml.jackson.core.type.TypeReference; -import java.util.Iterator; -import java.util.Optional; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.FieldType; @@ -418,14 +413,18 @@ public void all_query_returns_all_results() throws Exception { { List request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference>() { }); - Iterator response = dao.getAllLatest(request).iterator(); - Document bro2 = response.next(); - Assert.assertEquals("bro_1", bro2.getDocument().get("guid")); - Assert.assertEquals("bro", bro2.getDocument().get("source:type")); - Document snort2 = response.next(); - Assert.assertEquals("bro_2", snort2.getDocument().get("guid")); - Assert.assertEquals("bro", snort2.getDocument().get("source:type")); - Assert.assertFalse(response.hasNext()); + Map docs = new HashMap<>(); + + for(Document doc : dao.getAllLatest(request)) { + docs.put(doc.getGuid(), doc); + } + Assert.assertEquals(2, docs.size()); + Assert.assertTrue(docs.keySet().contains("bro-1")); + Assert.assertTrue(docs.keySet().contains("bro-2")); + for(Map.Entry kv : docs.entrySet()) { + Document d = kv.getValue(); + Assert.assertEquals("bro", d.getDocument().get("source:type")); + } } //Filter test case { From 14fee9960a8339611675db7cd04cf4da916192ce Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Thu, 16 Nov 2017 14:56:46 -0700 Subject: [PATCH 39/59] Fix meta alert tests. Deprecate the data pruner --- .../scripts/prune_elasticsearch_indices.sh | 3 +- ...ElasticsearchMetaAlertIntegrationTest.java | 29 ++++++++++++------- .../indexing/dao/SearchIntegrationTest.java | 13 +++++---- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh b/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh index c3f1d05f51..f891fa354a 100644 --- a/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh +++ b/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh @@ -17,5 +17,6 @@ # limitations under the License. # -yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.ElasticsearchDataPrunerRunner "$@" +echo "The Metron Elasticsearch data pruner has been deprecated in favor of the Curator framework." +echo "See https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/index.html" diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 416a516ea0..3cd8188c73 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -23,29 +23,36 @@ import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_FIELD; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; import static org.apache.metron.indexing.dao.MetaAlertDao.STATUS_FIELD; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; - -import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao; -import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.MetaAlertDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.Group; import org.apache.metron.indexing.dao.search.GroupRequest; @@ -157,7 +164,7 @@ public class ElasticsearchMetaAlertIntegrationTest { /** * { - "metaalert_doc" : { + "%MAPPING_NAME%_doc" : { "properties" : { "guid" : { "type" : "keyword" @@ -176,7 +183,7 @@ public class ElasticsearchMetaAlertIntegrationTest { } */ @Multiline - public static String templates; + public static String template; @BeforeClass public static void setupBefore() throws Exception { @@ -207,8 +214,8 @@ public static void setupBefore() throws Exception { @Before public void setup() throws IOException { - es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, - templates); + es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); + es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index")); } @AfterClass @@ -738,7 +745,7 @@ public void shouldSearchByNestedAlert() throws Exception { // Query against all indices. The child alert has no actual attached meta alerts, and should // be returned on its own. - searchResponse = metaDao.search(new SearchRequest() { + searchResponse = metaDao.search(new SearchRequest() { { setQuery( "(ip_src_addr:192.168.1.3 AND ip_src_port:8008)" diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index d1a58741a1..674709d9ba 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -17,9 +17,14 @@ */ package org.apache.metron.indexing.dao; -import java.util.*; - import com.fasterxml.jackson.core.type.TypeReference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.FieldType; @@ -31,11 +36,9 @@ import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; -import org.apache.metron.integration.InMemoryComponent; -import org.junit.AfterClass; -import org.apache.metron.indexing.dao.search.GroupResult; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.integration.InMemoryComponent; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; From 02f500b346ccd74ebb4146f73da056ebba394d56 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 17 Nov 2017 15:20:32 -0500 Subject: [PATCH 40/59] Fixing test merge error. --- .../integration/ElasticsearchSearchIntegrationTest.java | 4 ++-- .../org/apache/metron/indexing/dao/SearchIntegrationTest.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 8533cb589f..ebe18486a1 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -112,8 +112,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "median" : { "type": "keyword" }, * "max": { "type": "keyword" }, * "count": { "type": "keyword" }, - * "sum": { "type": "keyword" } - * "source:type": { "type": "text" }, + * "sum": { "type": "keyword" }, + * "source:type": { "type": "text" } * } * } * } diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 2a3da37ea4..86f401aff8 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -755,7 +755,8 @@ public void returns_column_data_for_multiple_indices() throws Exception { Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); + //NOTE: This is because the field is in both bro and snort and they have different types. + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field")); } @Test From e8bd47eaad103a2eef2a1ef715c07a90b65d4fc0 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 17 Nov 2017 16:27:54 -0500 Subject: [PATCH 41/59] Debug guards. --- .../metron/elasticsearch/dao/ElasticsearchDao.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index efc9eca67a..5c8244c809 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -183,7 +183,7 @@ protected SearchResponse search(SearchRequest request, QueryBuilder queryBuilder private org.elasticsearch.action.search.SearchRequest buildSearchRequest( SearchRequest searchRequest, QueryBuilder queryBuilder) throws InvalidSearchException { - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Got search request; request={}", ElasticsearchUtils.toJSON(searchRequest).orElse("???")); } SearchSourceBuilder searchBuilder = new SearchSourceBuilder() @@ -245,7 +245,9 @@ private org.elasticsearch.action.search.SearchRequest buildSearchRequest( // return the search request String[] indices = wildcardIndices(searchRequest.getIndices()); - LOG.debug("Built Elasticsearch request; indices={}, request={}", indices, searchBuilder.toString()); + if (LOG.isDebugEnabled()) { + LOG.debug("Built Elasticsearch request; indices={}, request={}", indices, searchBuilder.toString()); + } return new org.elasticsearch.action.search.SearchRequest() .indices(indices) .source(searchBuilder); @@ -290,7 +292,9 @@ private SearchResponse buildSearchResponse( searchResponse.setFacetCounts(getFacetCounts(facetFields, esResponse.getAggregations(), commonColumnMetadata )); } - LOG.debug("Built search response; response={}", ElasticsearchUtils.toJSON(searchResponse).orElse("???")); + if (LOG.isDebugEnabled()) { + LOG.debug("Built search response; response={}", ElasticsearchUtils.toJSON(searchResponse).orElse("???")); + } return searchResponse; } From d3b86c4157aece127cc8e8a723a0a824cd509033 Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 17 Nov 2017 16:33:35 -0500 Subject: [PATCH 42/59] String to Text --- .../elasticsearch/dao/ElasticsearchColumnMetadataDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java index 8e210b40b8..2fe6bb8f56 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java @@ -48,7 +48,7 @@ public class ElasticsearchColumnMetadataDao implements ColumnMetadataDao { private static Map elasticsearchTypeMap; static { Map fieldTypeMap = new HashMap<>(); - fieldTypeMap.put("string", FieldType.STRING); + fieldTypeMap.put("string", FieldType.TEXT); fieldTypeMap.put("ip", FieldType.IP); fieldTypeMap.put("integer", FieldType.INTEGER); fieldTypeMap.put("long", FieldType.LONG); From 1c45c21f6ed60d49bff051a02db665144c35df7d Mon Sep 17 00:00:00 2001 From: cstella Date: Fri, 17 Nov 2017 16:53:52 -0500 Subject: [PATCH 43/59] One more string to text miss. --- .../apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java index 8ee5f4b7b4..a6b69ba6ca 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java @@ -149,7 +149,7 @@ public void searchShouldSortByGivenFields() throws Exception { JSONObject sortBy = (JSONObject) aSortField.get("sortByStringDesc"); assertEquals("desc", sortBy.get("order")); assertEquals("_last", sortBy.get("missing")); - assertEquals("string", sortBy.get("unmapped_type")); + assertEquals("text", sortBy.get("unmapped_type")); } { // sort by integer ascending From fca9a39f69723c82ba9b812a757086bb817ab5ad Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Tue, 21 Nov 2017 16:44:58 -0700 Subject: [PATCH 44/59] Add Elasticsearch and Kibana documentation for developers and users. --- metron-deployment/README.md | 91 +++++++- metron-deployment/packaging/ambari/README.md | 209 +++++++++++++++++ .../metron-elasticsearch/README.md | 219 ++++++++++++++++++ 3 files changed, 514 insertions(+), 5 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 133f5233aa..f3844219fb 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -1,3 +1,16 @@ +# Metron Deployment + +## Table of Contents + +* [Overview](#overview) +* [Prerequisites](#prerequisites) +* [Ambari](#ambari) +* [Vagrant](#vagrant) +* [Ambari Management Pack](#ambari-management-pack) +* [RPMs](#rpms) +* [Kibana Dashboards](#kibana-dashboards) +* [Kerberos](#kerberos) + # Overview This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster containing Metron services using Ansible. These playbooks target RHEL/CentOS 6.x operating systems. @@ -146,19 +159,87 @@ rpm -i ## Kibana Dashboards +The dashboards installed by the Kibana custom action are managed by two JSON files: +* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template +* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json + +The first file, `kibana.template`, is an Elasticsearch template that specifies the proper mapping types for the Kibana index. This configuration is necessary due to a bug +in the default dynamic mappings provided by Elasticsearch for long types versus integer that are incompatible with Kibana \[1\]. The second file, `dashboard-bulkload.json`, +contains all of the dashboard metadata necessary to create the Metron dashboard. It is an Elasticsearch bulk-insert formatted file \[2\] that contains a series +of documents necessary for setting up the dashboard in Elasticsearch. The main features installed are index patterns, searches, and a variety of visualizations +that are used in the Metron dashboard. + +Deploying the existing dashboard is easy. Once the MPack is installed, run the Kibana service's action "Load Template" to install dashboards. This will no longer overwrite +the .kibana in Elasticsearch. The bulk load is configured to fail inserts for existing documents. If you want to _completely_ reload the dashboard, you would need to delete +the .kibana index and reload again from Ambari. + +1. [https://github.com/elastic/kibana/issues/9888#issuecomment-298096954](https://github.com/elastic/kibana/issues/9888#issuecomment-298096954) +2. [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docs-bulk.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docs-bulk.html) + +### Modifying Pre-Installed Dashboards + +You can modify dashboards in Kibana and bring those changes into the core MPack distribution by performing the following steps: + +1. Export the .kibana index from ES +2. Convert the data into the ES bulk load format +3. Replace the dashboard-bulkload.json file in the Kibana MPack. + +You can export the .kibana index using a tool like [https://github.com/taskrabbit/elasticsearch-dump] (https://github.com/taskrabbit/elasticsearch-dump). The important +feature is to have one document per line. Here's an exmaple export using elasticsearch-dump + +``` +elasticdump \ + --input=http://node1:9200/.kibana \ + --output=~/dashboard-data.json \ + --type=data +``` + +Once you've exported the data, you can now format it as a bulk load ES file by running the import/export tool located in +metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java. This tool can be run from full-dev +as follows + +``` +java -cp $METRON_HOME/lib/metron-elasticsearch-0.4.2-uber.jar org.apache.metron.elasticsearch.bulk.ElasticsearchImportExport \ + ~/dashboard-data.json \ + ~/dashbaord-bulkload.json +``` + +Now copy this file to the Kibana MPack, overwriting the existing bulk load file. That should be everything needed to backup the dashboard. + +**Note**: the dashboard Python Pickle binary file is deprecated and no longer used for backing up and restoring Kibana dashboards. The tooling is still provided as of this +version but is expected to be removed in the future. A section describing the deprecated backup process remains below. + +### Deprecated Dashboard Install/Backup Instructions + The dashboards installed by the Kibana custom action are managed by the dashboard.p file. This file is created by exporting existing dashboards from a running Kibana instance. -To create a new version of the file, make any necessary changes to Kibana (e.g. on quick-dev), and export with the appropriate script. +To create a new version of the file, make any necessary changes to Kibana (e.g. on full-dev), and export with the appropriate script. +**Script Options** ``` -python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py \ +[elasticsearch_host] ES host +[elasticsearch_port] ES port number +[input_output_filename] Filename used for reading or writing out pickle file +[-s] Flag to indicate that the .kibana index should be saved locally. Not including this flag will overwrite the .kibana + index completely with the contents of 'input_output_filename'. Careful with this. +``` + +**Saving a Backup** +``` +python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py \ $ES_HOST 9200 \ -packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p -s +~/dashboard.p -s ``` -Build the Ambari Mpack to get the dashboard updated appropriately. +**Restoring From a Backup** +``` +python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py \ +$ES_HOST 9200 \ +~/dashboard.p +``` -Once the MPack is installed, run the Kibana service's action "Load Template" to install dashboards. This will completely overwrite the .kibana in Elasticsearch, so use with caution. +**Note**: This method of writing the Kibana dashboard to Elasticsearch will overwrite the entire .kibana index. Be sure to first backup the index first using either the new JSON +method described above, or writing out the dashboard.p pickle file using the old method (passing -s option to dashboardindex.py) described here. ## Kerberos The MPack can allow Metron to be installed and then Kerberized, or installed on top of an already Kerberized cluster. This is done through Ambari's standard Kerberization setup. diff --git a/metron-deployment/packaging/ambari/README.md b/metron-deployment/packaging/ambari/README.md index af511b4d51..f3eb4d1950 100644 --- a/metron-deployment/packaging/ambari/README.md +++ b/metron-deployment/packaging/ambari/README.md @@ -10,6 +10,7 @@ * [Configuration involving dependency services](#configuration-involving-dependency-services) * [Kerberos](#kerberos) * [Best practices](#best-practices) +* [Upgrading MPack Services](#upgrading-mpack-services) ## Overview Typically, Ambari Management Pack development will be done in the Vagrant environments. These instructions are specific to Vagrant, but can be adapted for other environemnts (e.g. make sure to be on the correct nodes for server vs agent files) @@ -447,3 +448,211 @@ This is checked in the indexing master * Make sure to `kinit` as the correct user for setting up ACLs in a secured cluster. This is usually kafka for Kafka and hbase for HBase. * See `set_hbase_acls` in `METRON.CURRENT/package/scripts/enrichment_commands.py` for an HBase example * See `init_kafka_acls` in `METRON.CURRENT/package/scripts/enrichment_commands.py` and `METRON.CURRENT/package/scripts/metron_service.py` for an Kafka example + +## Upgrading MPack Services + +Apache Metron currently provides three services as part of its MPack +* Elasticsearch +* Kibana +* Metron + +There is currently no mechanism provided for multi-version or backwards compatibility. If you upgrade a service, e.g. Elasticsearch 2.x to 5.x, that is the only version that will be +supported by Ambari via MPack. + +The main steps for upgrading a service are split into add-on and common services for each service within the MPack as follows: +* Update the add-on services + * Change the service directory to use the new product version number + * Update repoinfo.xml + * Update metainfo.xml +* Update the common services + * Change the service directory to use the new product version number + * Update metainfo.xml +* Update mpack.json + +### Update Elasticsearch + +#### Update Add-on Services + +1. Change service directory names for Elasticsearch to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2 + ``` + +1. Update repoinfo.xml + + See [https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html](https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html) for the latest info. + + Modify the baseurl and repoid in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/repos/repoinfo.xml`, e.g. + + ``` + https://artifacts.elastic.co/packages/5.x/yum + elasticsearch-5.x + ELASTICSEARCH + ``` + +1. Update metainfo.xml + + Change the version number in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`. + Also make sure to update the "extends" version to point to the updated common-services version, e.g. + + ``` + ELASTICSEARCH + 5.6.2 + common-services/ELASTICSEARCH/5.6.2 + ``` + +#### Update Common Services + +1. Change service directory names for Elasticsearch to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2 + ``` + +1. Update metainfo.xml + + Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g. + + ``` + 5.6.2 + ... + + + any + + + elasticsearch-5.6.2 + + + + + ``` + +#### Update mpack.json + +1. Update the corresponding service_version in the service_versions_map, e.g. + + ``` + ... + "service_versions_map": [ + { + "service_name" : "ELASTICSEARCH", + "service_version" : "5.6.2", + "applicable_stacks" : [ + ... + ] + }, + ... + ] + ... + ``` + +### Kibana + +**Note:** Curator is included with the Kibana service + +#### Update Add-on Services + +1. Change service directory names for Kibana to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2 + ``` + +1. Update repoinfo.xml + + **Note:** for Curator, there is a different repo for rhel 6 vs rhel 7 + + See the following links for current repo information for Kibana and Curator. + * [https://www.elastic.co/guide/en/kibana/current/rpm.html](https://www.elastic.co/guide/en/kibana/current/rpm.html) + * [https://www.elastic.co/guide/en/elasticsearch/client/curator/current/yum-repository.html](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/yum-repository.html) + + Modify the baseurl's and repoid's in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/repos/repoinfo.xml`, e.g. + + ``` + https://artifacts.elastic.co/packages/5.x/yum + kibana-5.x + KIBANA + ... + http://packages.elastic.co/curator/5/centos/6 + ES-Curator-5.x + CURATOR + ``` + +1. Update metainfo.xml + + Change the version number in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`. + Also make sure to update the "extends" version to point to the updated common-services version, e.g. + ``` + KIBANA + 5.6.2 + common-services/KIBANA/5.6.2 + ``` + +#### Update Common Services + +1. Change service directory names for Kibana to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2 + ``` + +1. Update metainfo.xml + + Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g. + + ``` + 5.6.2 + ... + + ... + + kibana-5.6.2 + + + ``` + +#### Update mpack.json + +1. Update the corresponding service_version in the service_versions_map, e.g. + + ``` + ... + "service_versions_map": [ + { + "service_name" : "KIBANA", + "service_version" : "5.6.2", + "applicable_stacks" : [ + ... + ] + }, + ... + ] + ... + ``` + diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md index 9113672cc6..ca374e6fee 100644 --- a/metron-platform/metron-elasticsearch/README.md +++ b/metron-platform/metron-elasticsearch/README.md @@ -1,5 +1,13 @@ # Elasticsearch in Metron +* [Table of Contents](#table-of-contents) +* [Introduction](#introduction) +* [Properties](#properties) +* [Upgrading to 5.x](#upgrading-to-5x) +* [Type Mappings](#type-mappings) +* [Using Metron with Elasticsearch 2.x](#using-metron-with-elasticsearch-2x) +* [Installing Elasticsearch Templates](#installing-elasticsearch-templates) + ## Introduction Elasticsearch can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md). @@ -33,6 +41,217 @@ For instance, an `es.date.format` of `yyyy.MM.dd.HH` would have the consequence roll hourly, whereas an `es.date.format` of `yyyy.MM.dd` would have the consequence that the indices would roll daily. +## Upgrading to 5.x + +Users should be prepared to re-index when migrating from Elasticsearch 2.x to 5.x. There are a number of template changes, most notably around +string type handling, that may cause issues when upgrading. + +[https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html) + +## Type Mappings + +Type mappings have changed quite a bit from ES 2.x -> 5.x. Here is a brief rundown of the biggest changes. More detailed references from Elasticsearch +are provided in the [Type Mapping References](#type-mapping-references) section below. +* string fields replaced by text/keyword type. +* strings have new default mappings as follows + ``` + { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + ``` +* There is no longer a `_timestamp` field that you can set "enabled" on. This field now causes an exception on templates. +Replace with an application-created timestamp of "date" type. + +The semantics for string types have changed. In 2.x, you have the concept of index settings as either "analyzed" or "not_analyzed" which basically means "full text" and "keyword", respectively. +Analyzed text basically means the indexer will split the text using a text analyzer thus allowing you to search on substrings within the original text. "New York" is split and indexed as two buckets, + "New" and "York", so you can search or query for aggregate counts for those terms independently and will match against the individual terms "New" or "York." "Keyword" means that the original text + will not be split/analyzed during indexing and instead treated as a whole unit, i.e. "New" or "York" will not match in searches against the document containing "New York", but searching on "New York" + as the full city name will. In 5.x language instead of using the "index" setting, you now set the "type" to either "text" for full text, or "keyword" for keywords. + +Below is a table depicting the changes to how String types are now handled. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
sort, aggregate, or access valuesES 2.xES 5.xExample
no +

+"my_property" : {
+  "type": "string",
+  "index": "analyzed"
+}
+
+ 		+

+"my_property" : {
+  "type": "text"
+}
+
+ Additional defaults: "index": "true", "fielddata": "false" + 		+ "New York" handled via in-mem search as "New" and "York" buckets. No aggregation or sort. +
+ yes + +

+"my_property": {
+  "type": "string",
+  "index": "analyzed"
+}
+
+ 		+

+"my_property": {
+  "type": "text",
+  "fielddata": "true"
+}
+
+ 		+ "New York" handled via in-mem search as "New" and "York" buckets. Can aggregate and sort. +
+ yes + +

+"my_property": {
+  "type": "string",
+  "index": "not_analyzed"
+}
+
+ 		+

+"my_property" : {
+  "type": "keyword"
+}
+
+ 		+ "New York" searchable as single value. Can aggregate and sort. A search for "New" or "York" will not match against the whole value. +
+ yes + +

+"my_property": {
+  "type": "string",
+  "index": "analyzed"
+}
+
+ 		+

+"my_property": {
+  "type": "text",
+  "fields": {
+    "keyword": {
+      "type": "keyword",
+      "ignore_above": 256
+    }
+  }
+}
+
+ 		+ "New York" searchable as single value or as text document, can aggregate and sort on the sub term "keyword." +
+ +If you want to set default string behavior for all strings for a given index and type, you can do so with a mapping similar to the following (replace ${your_type_here} accordingly): +``` +# curl -XPUT 'http://${ES_HOST}:${ES_PORT}/_template/default_string_template' -d ' +{ + "template": "*", + "mappings" : { + "${your_type_here}": { + "dynamic_templates": [ + { + "strings": { + "match_mapping_type": "string", + "mapping": { + "type": "text" + } + } + } + ] + } + } +} +' +``` + +By specifying the "template" property with value "*" the template will apply to all indexes that have documents indexed of the specified type (${your_type_here}). This results in the following template. +``` +# curl -XGET 'http://${ES_HOST}:${ES_PORT}/_template/default_string_template?pretty' +{ + "default_string_template" : { + "order" : 0, + "template" : "*", + "settings" : { }, + "mappings" : { + "${your_type_here}" : { + "dynamic_templates" : [ + { + "strings" : { + "match_mapping_type" : "string", + "mapping" : { + "type" : "text" + } + } + } + ] + } + }, + "aliases" : { } + } +} +``` + +Notes on other settings for types in ES +* doc_values + * on-disk data structure + * provides access for sorting, aggregation, and field values + * stores same values as _source, but in column-oriented fashion better for sorting and aggregating + * not supported on text fields + * enabled by default +* fielddata + * in-memory data structure + * provides access for sorting, aggregation, and field values + * primarily for text fields + * disabled by default because the heap space required can be large + + +##### Type Mapping References +* [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html) +* [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html) +* [https://www.elastic.co/blog/strings-are-dead-long-live-strings](https://www.elastic.co/blog/strings-are-dead-long-live-strings) + ## Using Metron with Elasticsearch 2.x With Elasticsearch 2.x, there is a requirement that all sensors templates have a nested alert field defined. This field is a dummy field, and will be obsolete in Elasticsearch 5.x. See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information From 307ddc4afc877a35085da00645c40782f315b01e Mon Sep 17 00:00:00 2001 From: Michael Miklavcic Date: Wed, 22 Nov 2017 13:32:01 -0700 Subject: [PATCH 45/59] Fix docs. Change Kibana all index chart colors --- metron-deployment/packaging/ambari/README.md | 130 +++++++++--------- .../scripts/dashboard/dashboard-bulkload.json | 2 +- .../metron-elasticsearch/README.md | 84 ++++++----- 3 files changed, 106 insertions(+), 110 deletions(-) diff --git a/metron-deployment/packaging/ambari/README.md b/metron-deployment/packaging/ambari/README.md index f3eb4d1950..32ea85d36d 100644 --- a/metron-deployment/packaging/ambari/README.md +++ b/metron-deployment/packaging/ambari/README.md @@ -460,17 +460,50 @@ There is currently no mechanism provided for multi-version or backwards compatib supported by Ambari via MPack. The main steps for upgrading a service are split into add-on and common services for each service within the MPack as follows: -* Update the add-on services +* Update the common services * Change the service directory to use the new product version number - * Update repoinfo.xml * Update metainfo.xml -* Update the common services +* Update the add-on services * Change the service directory to use the new product version number + * Update repoinfo.xml * Update metainfo.xml * Update mpack.json ### Update Elasticsearch +#### Update Common Services + +1. Change service directory names for Elasticsearch to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2 + ``` + +1. Update metainfo.xml + + Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g. + + ``` + 5.6.2 + ... + + + any + + + elasticsearch-5.6.2 + + + + + ``` + #### Update Add-on Services 1. Change service directory names for Elasticsearch to the new desired version @@ -508,39 +541,6 @@ The main steps for upgrading a service are split into add-on and common services common-services/ELASTICSEARCH/5.6.2 ``` -#### Update Common Services - -1. Change service directory names for Elasticsearch to the new desired version - - ``` - metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE} - ``` - - e.g. - - ``` - metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2 - ``` - -1. Update metainfo.xml - - Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g. - - ``` - 5.6.2 - ... - - - any - - - elasticsearch-5.6.2 - - - - - ``` - #### Update mpack.json 1. Update the corresponding service_version in the service_versions_map, e.g. @@ -564,6 +564,35 @@ The main steps for upgrading a service are split into add-on and common services **Note:** Curator is included with the Kibana service +#### Update Common Services + +1. Change service directory names for Kibana to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2 + ``` + +1. Update metainfo.xml + + Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g. + + ``` + 5.6.2 + ... + + ... + + kibana-5.6.2 + + + ``` + #### Update Add-on Services 1. Change service directory names for Kibana to the new desired version @@ -608,35 +637,6 @@ The main steps for upgrading a service are split into add-on and common services common-services/KIBANA/5.6.2 ``` -#### Update Common Services - -1. Change service directory names for Kibana to the new desired version - - ``` - metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE} - ``` - - e.g. - - ``` - metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2 - ``` - -1. Update metainfo.xml - - Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g. - - ``` - 5.6.2 - ... - - ... - - kibana-5.6.2 - - - ``` - #### Update mpack.json 1. Update the corresponding service_version in the service_versions_map, e.g. diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json index dfadd48a00..c6087136ca 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json @@ -11,7 +11,7 @@ { "create" : { "_id": "AV-YyJw3PfR7HJex-ZdY", "_type": "visualization" } } {"title":"All index TS event count","visState":"{\"title\":\"All index TS event count\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"metric\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Event Count\",\"split_filters\":[{\"color\":\"#68BC00\",\"id\":\"89be23f0-c4af-11e7-ac01-25d5c1ff2e49\"}],\"series_drop_last_bucket\":0}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro_index*,snort_index*,yaf_index*\",\"interval\":\"1y\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0f0-c412-11e7-a0b9-2137696bd057\"}],\"filter\":\"\",\"drop_last_bucket\":0},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} { "create" : { "_id": "AV-cBm5JFLIoshSSHghu", "_type": "visualization" } } -{"title":"All index TS Chart","visState":"{\"title\":\"All index TS Chart\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"timeseries\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":\"1\",\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Events\",\"terms_field\":\"source:type\",\"value_template\":\"{{value}}\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro*,snort*,yaf*\",\"interval\":\"30s\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0f0-c412-11e7-a0b9-2137696bd057\"}],\"show_grid\":1,\"drop_last_bucket\":0},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} +{"title":"All index TS Chart","visState":"{\"title\":\"All index TS Chart\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"timeseries\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":\"1\",\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Events\",\"terms_field\":\"source:type\",\"value_template\":\"{{value}}\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro*,snort*,yaf*\",\"interval\":\"30s\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0f0-c412-11e7-a0b9-2137696bd057\"}],\"show_grid\":1,\"drop_last_bucket\":0},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}} { "create" : { "_id": "AV-dXz9Lk7f2nZ-iH3Kb", "_type": "visualization" } } {"title":"Event Count Pie Chart","visState":"{\"title\":\"Event Count Pie Chart\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Events by Source Type\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source:type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}} { "create" : { "_id": "AV-ddhh7k7f2nZ-iH3Kx", "_type": "visualization" } } diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md index ca374e6fee..9e7fe05bf5 100644 --- a/metron-platform/metron-elasticsearch/README.md +++ b/metron-platform/metron-elasticsearch/README.md @@ -52,19 +52,21 @@ string type handling, that may cause issues when upgrading. Type mappings have changed quite a bit from ES 2.x -> 5.x. Here is a brief rundown of the biggest changes. More detailed references from Elasticsearch are provided in the [Type Mapping References](#type-mapping-references) section below. -* string fields replaced by text/keyword type. +* string fields replaced by text/keyword type * strings have new default mappings as follows - ``` - { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + + ``` + { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } } } - } - ``` + ``` + * There is no longer a `_timestamp` field that you can set "enabled" on. This field now causes an exception on templates. Replace with an application-created timestamp of "date" type. @@ -86,16 +88,14 @@ Below is a table depicting the changes to how String types are now handled. no -

-"my_property" : {
+
"my_property" : {
   "type": "string",
   "index": "analyzed"
 }
 

 	
 	
-

-"my_property" : {
+
"my_property" : {
   "type": "text"
 }
 

@@ -110,16 +110,14 @@ Below is a table depicting the changes to how String types are now handled.
 	yes
 	
 	
-

-"my_property": {
+
"my_property": {
   "type": "string",
   "index": "analyzed"
 }
 

 	
 	
-

-"my_property": {
+
"my_property": {
   "type": "text",
   "fielddata": "true"
 }
@@ -134,16 +132,14 @@ Below is a table depicting the changes to how String types are now handled.
 	yes
 	
 	
-

-"my_property": {
+
"my_property": {
   "type": "string",
   "index": "not_analyzed"
 }
 

 	
 	
-

-"my_property" : {
+
"my_property" : {
   "type": "keyword"
 }
 

@@ -157,16 +153,14 @@ Below is a table depicting the changes to how String types are now handled.
 	yes
 	
 	
-

-"my_property": {
+
"my_property": {
   "type": "string",
   "index": "analyzed"
 }
 

 	
 	
-

-"my_property": {
+
"my_property": {
   "type": "text",
   "fields": {
     "keyword": {
@@ -184,29 +178,31 @@ Below is a table depicting the changes to how String types are now handled.
 
 
 If you want to set default string behavior for all strings for a given index and type, you can do so with a mapping similar to the following (replace ${your_type_here} accordingly):
+
 ```
 # curl -XPUT 'http://${ES_HOST}:${ES_PORT}/_template/default_string_template' -d '
 {
-    "template": "*",
-    "mappings" : {
-        "${your_type_here}": {
-            "dynamic_templates": [
-                {
-                    "strings": {
-                        "match_mapping_type": "string",
-                        "mapping": {
-                            "type": "text"
-                        }
-                    }
-                }
-            ]
+  "template": "*",
+  "mappings" : {
+    "${your_type_here}": {
+      "dynamic_templates": [
+        {
+          "strings": {
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text"
+            }
+          }
         }
+      ]
     }
+  }
 }
 '
 ```
 
 By specifying the "template" property with value "*" the template will apply to all indexes that have documents indexed of the specified type (${your_type_here}). This results in the following template.
+
 ```
 # curl -XGET 'http://${ES_HOST}:${ES_PORT}/_template/default_string_template?pretty'
 {
@@ -291,11 +287,11 @@ To update existing indexes, update Elasticsearch mappings with the new field for
 ```
 curl -XPUT "http://${ELASTICSEARCH}:9200/${SENSOR}_index*/_mapping/${SENSOR}_doc" -d '
 {
-        "properties" : {
-          "alert" : {
-            "type" : "nested"
-          }
-        }
+  "properties" : {
+    "alert" : {
+      "type" : "nested"
+    }
+  }
 }
 '
 rm ${SENSOR}.template

From 00e426a98b8156d18da17d0f92302a74554d2740 Mon Sep 17 00:00:00 2001
From: cstella 
Date: Mon, 27 Nov 2017 11:42:14 -0500
Subject: [PATCH 46/59] Fixing tests.

---
 .../metron/elasticsearch/dao/ElasticsearchDao.java     |  3 +++
 .../metron/elasticsearch/utils/ElasticsearchUtils.java |  6 ++++--
 .../metron/elasticsearch/dao/ElasticsearchDaoTest.java |  2 +-
 .../dao/ElasticsearchRequestSubmitterTest.java         | 10 +++++-----
 4 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 5c8244c809..650462e5d7 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -147,6 +147,9 @@ public ElasticsearchDao() {
 
   @Override
   public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
+    if(searchRequest.getQuery() == null) {
+      throw new InvalidSearchException("Search query is invalid: null");
+    }
     return search(searchRequest, new QueryStringQueryBuilder(searchRequest.getQuery()));
   }
 
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
index 318f56f8ae..f948e398a4 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
@@ -24,6 +24,7 @@
 import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -202,9 +203,10 @@ else if(ipObj instanceof List) {
   public static Optional toJSON(org.elasticsearch.action.search.SearchRequest esRequest) {
     Optional json = Optional.empty();
 
-    if(esRequest != null) {
+    if(esRequest != null && esRequest.source() != null) {
       try {
-        json = Optional.of(XContentHelper.convertToJson(esRequest.source().buildAsBytes(), true));
+        BytesReference requestBytes = esRequest.source().buildAsBytes();
+        json = Optional.of(XContentHelper.convertToJson(requestBytes, true));
 
       } catch (Throwable t) {
         LOG.error("Failed to convert search request to JSON", t);
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
index a6b69ba6ca..2a6fb4ff8e 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
@@ -218,7 +218,7 @@ public void searchShouldThrowExceptionWhenMaxResultsAreExceeded() throws Excepti
 
     SearchRequest searchRequest = new SearchRequest();
     searchRequest.setSize(maxSearchResults+1);
-
+    searchRequest.setQuery("");
     dao.search(searchRequest);
     // exception expected - size > max
   }
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitterTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitterTest.java
index 26f5fff7f1..07019c3b50 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitterTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitterTest.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchShardTarget;
 import org.junit.Test;
@@ -56,7 +57,7 @@ public void searchShouldSucceedWhenOK() throws InvalidSearchException {
 
     // mocks
     SearchResponse response = mock(SearchResponse.class);
-    SearchRequest request = mock(SearchRequest.class);
+    SearchRequest request = new SearchRequest();
 
     // response will have status of OK and no failed shards
     when(response.status()).thenReturn(RestStatus.OK);
@@ -74,7 +75,7 @@ public void searchShouldFailWhenNotOK() throws InvalidSearchException {
 
     // mocks
     SearchResponse response = mock(SearchResponse.class);
-    SearchRequest request = mock(SearchRequest.class);
+    SearchRequest request = new SearchRequest();
 
     // response will have status of OK
     when(response.status()).thenReturn(RestStatus.PARTIAL_CONTENT);
@@ -90,9 +91,9 @@ public void searchShouldFailWhenNotOK() throws InvalidSearchException {
   public void searchShouldHandleShardFailure() throws InvalidSearchException {
     // mocks
     SearchResponse response = mock(SearchResponse.class);
-    SearchRequest request = mock(SearchRequest.class);
+    SearchRequest request = new SearchRequest();
     ShardSearchFailure fail = mock(ShardSearchFailure.class);
-    SearchShardTarget target = mock(SearchShardTarget.class);
+    SearchShardTarget target = new SearchShardTarget("node1", mock(Index.class), 1, "metron");
 
     // response will have status of OK
     when(response.status()).thenReturn(RestStatus.OK);
@@ -107,7 +108,6 @@ public void searchShouldHandleShardFailure() throws InvalidSearchException {
 
     // shard failure needs to report the node
     when(fail.shard()).thenReturn(target);
-    when(target.getNodeId()).thenReturn("node1");
 
     // shard failure needs to report details of failure
     when(fail.index()).thenReturn("bro_index_2017-10-11");

From 4581ad5f4ff3ffcde823057061b63abc1cf81bfb Mon Sep 17 00:00:00 2001
From: cstella 
Date: Mon, 27 Nov 2017 12:39:50 -0500
Subject: [PATCH 47/59] Fixing integration test.

---
 .../elasticsearch/dao/ElasticsearchColumnMetadataDao.java | 3 ++-
 .../integration/ElasticsearchSearchIntegrationTest.java   | 6 +++---
 .../apache/metron/indexing/dao/SearchIntegrationTest.java | 8 ++++----
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
index 2fe6bb8f56..c2e2627ab5 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
@@ -48,7 +48,8 @@ public class ElasticsearchColumnMetadataDao implements ColumnMetadataDao {
   private static Map elasticsearchTypeMap;
   static {
     Map fieldTypeMap = new HashMap<>();
-    fieldTypeMap.put("string", FieldType.TEXT);
+    fieldTypeMap.put("text", FieldType.TEXT);
+    fieldTypeMap.put("keyword", FieldType.KEYWORD);
     fieldTypeMap.put("ip", FieldType.IP);
     fieldTypeMap.put("integer", FieldType.INTEGER);
     fieldTypeMap.put("long", FieldType.LONG);
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index e139e69901..db31e95d39 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -64,7 +64,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
    *     },
    *     "guid" : {
    *        "type" : "keyword"
-   *     }
+   *     },
    *     "ip_src_addr": {
    *        "type": "ip"
    *     },
@@ -117,7 +117,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
    *        },
    *        "guid" : {
    *          "type" : "keyword"
-   *        }
+   *        },
    *        "ip_src_addr": {
    *          "type": "ip"
    *        },
@@ -179,7 +179,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
    *     "max": { "type": "keyword" },
    *     "count": { "type": "keyword" },
    *     "sum": { "type": "keyword" },
-   *     "source:type": { "type": "text" }
+   *     "source:type": { "type": "keyword" }
    *   }
    * }
    * }
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index ac2c6fdca5..62311b4544 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -829,7 +829,7 @@ public void facet_query_yields_field_types() throws Exception {
   @Test
   public void bad_facet_query_throws_exception() throws Exception {
     thrown.expect(InvalidSearchException.class);
-    thrown.expectMessage("Could not execute search");
+    thrown.expectMessage("Failed to execute search");
     SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class);
     dao.search(request);
   }
@@ -852,7 +852,7 @@ public void exceeding_max_resulsts_throws_exception() throws Exception {
   @Test
   public void returns_column_data_for_multiple_indices() throws Exception {
     Map fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort"));
-    Assert.assertEquals(13, fieldTypes.size());
+    Assert.assertEquals(15, fieldTypes.size());
     Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
     Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type"));
     Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
@@ -880,7 +880,7 @@ public void returns_column_metadata_for_specified_indices() throws Exception {
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
       Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
+      Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type"));
       Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
       Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
       Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
@@ -1086,7 +1086,7 @@ public void group_by_returns_results_in_sorted_groups() throws Exception {
   public void throws_exception_on_aggregation_queries_on_non_string_non_numeric_fields()
           throws Exception {
     thrown.expect(InvalidSearchException.class);
-    thrown.expectMessage("Could not execute search");
+    thrown.expectMessage("Failed to execute search");
     GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class);
     dao.group(request);
   }

From 9d7a4bd616506bed9ba319ae5c0b0616edfa7822 Mon Sep 17 00:00:00 2001
From: Michael Miklavcic 
Date: Mon, 27 Nov 2017 11:26:54 -0700
Subject: [PATCH 48/59] Add Curator documentation to metron-data-management
 README.

---
 .../metron-data-management/README.md          | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/metron-platform/metron-data-management/README.md b/metron-platform/metron-data-management/README.md
index c351f48e29..ee24e2df4e 100644
--- a/metron-platform/metron-data-management/README.md
+++ b/metron-platform/metron-data-management/README.md
@@ -1,5 +1,9 @@
 # Resource Data Management
 
+## Table of Contents
+
+## Overview
+
 This project is a collection of classes to assist with loading of
 various enrichment and threat intelligence sources into Metron.
 
@@ -337,3 +341,39 @@ The parameters for the utility are as follows:
 | -r         | --remote_dir        | No           | HDFS directory to land formatted GeoIP file - defaults to /apps/metron/geo/\/     |
 | -t         | --tmp_dir           | No           | Directory for landing the temporary GeoIP data - defaults to /tmp                                |
 | -z         | --zk_quorum         | Yes          | Zookeeper Quorum URL (zk1:port,zk2:port,...)                                                     |
+
+## Pruning Data from Elasticsearch
+
+**Note** - As of the Metron upgrade from Elasticsearch 2.x to 5.x, the included Data Pruner is no longer supported. It is replaced in favor of the Curator utility
+provided by Elasticsearch.
+
+Elasticsearch provides tooling to prune index data through [Curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/index.html).
+
+Here is a sample invocation that you can configure through Cron to prune indexes based on timestamp in the index name.
+
+```
+/opt/elasticsearch-curator/curator_cli --host localhost delete_indices --filter_list '
+    {
+      "filtertype": "age",
+      "source": "name",
+      "timestring": "%Y.%m.%d",
+      "unit": "days",
+      "unit_count": 10,
+      "direction": "older”
+    }'
+```
+
+From the ES documentation:
+> Using name as the source tells Curator to look for a timestring within the index or snapshot name, and convert that into an epoch timestamp (epoch implies UTC).
+
+You can also provide multiple filters as an array of JSON objects to filter_list if you want finer-grained control over the indexes that will be pruned.
+There is an implicit logical AND when chaining multiple filters.
+
+```
+--filter_list '[{"filtertype":"age","source":"creation_date","direction":"older","unit":"days","unit_count":13},{"filtertype":"pattern","kind":"prefix","value":"logstash"}]'
+```
+
+### Reference
+* [https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/index.html](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/index.html)
+* [https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/filtertype_age.html](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/filtertype_age.html)
+* [https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/singleton-cli.html](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/singleton-cli.html)

From 4a4871a436e71469c858e3b27a320bd8b5d7b21f Mon Sep 17 00:00:00 2001
From: cstella 
Date: Mon, 27 Nov 2017 14:30:04 -0500
Subject: [PATCH 49/59] Undoing bad merge.

---
 .../ElasticsearchSearchIntegrationTest.java           | 11 ++++++++---
 .../metron/indexing/dao/SearchIntegrationTest.java    |  8 ++++----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index db31e95d39..3dde41a655 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -60,7 +60,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
    * "bro_doc": {
    *   "properties": {
    *     "source:type": {
-   *        "type": "keyword"
+   *        "type": "text",
+   *        "fielddata" : "true"
    *     },
    *     "guid" : {
    *        "type" : "keyword"
@@ -113,7 +114,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
    *  "snort_doc": {
    *     "properties": {
    *        "source:type": {
-   *          "type": "keyword"
+   *          "type": "text",
+   *          "fielddata" : "true"
    *        },
    *        "guid" : {
    *          "type" : "keyword"
@@ -179,7 +181,10 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
    *     "max": { "type": "keyword" },
    *     "count": { "type": "keyword" },
    *     "sum": { "type": "keyword" },
-   *     "source:type": { "type": "keyword" }
+   *     "source:type": {
+   *       "type": "text",
+   *       "fielddata" : "true"
+   *                    }
    *   }
    * }
    * }
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 62311b4544..72e632fee4 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -755,7 +755,7 @@ public void returns_results_only_for_specified_indices() throws Exception {
   public void facet_query_yields_field_types() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
     SearchResponse response = dao.search(request);
-    Assert.assertEquals(10, response.getTotal());
+    Assert.assertEquals(12, response.getTotal());
     Map> facetCounts = response.getFacetCounts();
     Assert.assertEquals(8, facetCounts.size());
     Map sourceTypeCounts = facetCounts.get("source:type");
@@ -854,7 +854,7 @@ public void returns_column_data_for_multiple_indices() throws Exception {
     Map fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort"));
     Assert.assertEquals(15, fieldTypes.size());
     Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
-    Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type"));
+    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
     Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
     Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
     Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
@@ -880,7 +880,7 @@ public void returns_column_metadata_for_specified_indices() throws Exception {
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
       Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
-      Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type"));
+      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
       Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
       Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
       Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
@@ -899,7 +899,7 @@ public void returns_column_metadata_for_specified_indices() throws Exception {
       Assert.assertEquals(14, fieldTypes.size());
       Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
       Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
+      Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
       Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
       Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));

From 74a5422ddf5ca169e16d3b4b7af260e837c25cfe Mon Sep 17 00:00:00 2001
From: cstella 
Date: Mon, 27 Nov 2017 14:57:56 -0500
Subject: [PATCH 50/59] Fixed snort template

---
 .../METRON/CURRENT/package/files/snort_index.template    | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index fd394426a3..0d79b313b0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -114,15 +114,6 @@
             "match_mapping_type": "*"
           }
         }
-      },
-      {
-        "threat_triage_name": {
-          "mapping": {
-            "type": "string"
-          },
-          "match": "threat:triage:rules:*:name",
-          "match_mapping_type": "*"
-        }
       }
       ],
       "properties": {

From 4001edf577cf93a7f6c2f6a510fa16ba5e06b9e6 Mon Sep 17 00:00:00 2001
From: cstella 
Date: Mon, 27 Nov 2017 15:00:58 -0500
Subject: [PATCH 51/59] Snort template merge issue.

---
 .../METRON/CURRENT/package/files/snort_index.template            | 1 -
 1 file changed, 1 deletion(-)

diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index 0d79b313b0..0ddb631fd9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -114,7 +114,6 @@
             "match_mapping_type": "*"
           }
         }
-      }
       ],
       "properties": {
         "timestamp": {

From c4c88db615a36891de48f2333db5be965597cba2 Mon Sep 17 00:00:00 2001
From: cstella 
Date: Mon, 27 Nov 2017 15:07:48 -0500
Subject: [PATCH 52/59] Fixing test.

---
 .../metron/rest/controller/SearchControllerIntegrationTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
index 0db0e1ac37..d8758cdfa1 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
@@ -146,7 +146,7 @@ public void testColumnMetadataUsingDefaultIndices() throws Exception {
         .andExpect(status().isOk())
         .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
         .andExpect(jsonPath("$.*", hasSize(5)))
-        .andExpect(jsonPath("$.common_string_field").value("string"))
+        .andExpect(jsonPath("$.common_string_field").value("text"))
         .andExpect(jsonPath("$.common_integer_field").value("integer"))
         .andExpect(jsonPath("$.bro_field").value("boolean"))
         .andExpect(jsonPath("$.snort_field").value("double"))

From 9458a2095ba0cb158611028e03e0fe45eabaf785 Mon Sep 17 00:00:00 2001
From: Michael Miklavcic 
Date: Mon, 27 Nov 2017 15:58:08 -0700
Subject: [PATCH 53/59] Add TOC to metron-data-management. Specify 5.6.2 as ES
 version where specific version makes sense

---
 metron-platform/metron-data-management/README.md | 11 +++++++++--
 metron-platform/metron-elasticsearch/README.md   | 13 +++++++------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/metron-platform/metron-data-management/README.md b/metron-platform/metron-data-management/README.md
index ee24e2df4e..c514e2a928 100644
--- a/metron-platform/metron-data-management/README.md
+++ b/metron-platform/metron-data-management/README.md
@@ -2,6 +2,13 @@
 
 ## Table of Contents
 
+* [Overview](#overview)
+* [Simple HBase Enrichments/Threat Intelligence](#simple-hbase-enrichmentsthreat-intelligence)
+* [Extractor Framework](#extractor-framework)
+* [Enrichment Config](#enrichment-config)
+* [Loading Utilities](#loading-utilities)
+* [Pruning Data from Elasticsearch](#pruning-data-from-elasticsearch)
+
 ## Overview
 
 This project is a collection of classes to assist with loading of
@@ -344,8 +351,8 @@ The parameters for the utility are as follows:
 
 ## Pruning Data from Elasticsearch
 
-**Note** - As of the Metron upgrade from Elasticsearch 2.x to 5.x, the included Data Pruner is no longer supported. It is replaced in favor of the Curator utility
-provided by Elasticsearch.
+**Note** - As of the Metron upgrade from Elasticsearch 2.3.3 to 5.6.2, the included Data Pruner is no longer supported. It is replaced in favor of the Curator utility
+provided by Elasticsearch. The current Curator version is 5.4 as of this version of Metron and does not match exactly with ES and Kibana.
 
 Elasticsearch provides tooling to prune index data through [Curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.4/index.html).
 
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index 9e7fe05bf5..d235d0935b 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -1,11 +1,12 @@
 # Elasticsearch in Metron
 
-* [Table of Contents](#table-of-contents)
+## Table of Contents
+
 * [Introduction](#introduction)
 * [Properties](#properties)
-* [Upgrading to 5.x](#upgrading-to-5x)
+* [Upgrading to 5.6.2](#upgrading-to-562)
 * [Type Mappings](#type-mappings)
-* [Using Metron with Elasticsearch 2.x](#using-metron-with-elasticsearch-2x)
+* [Using Metron with Elasticsearch 5.x](#using-metron-with-elasticsearch-5x)
 * [Installing Elasticsearch Templates](#installing-elasticsearch-templates)
 
 ## Introduction
@@ -41,9 +42,9 @@ For instance, an `es.date.format` of `yyyy.MM.dd.HH` would have the consequence
 roll hourly, whereas an `es.date.format` of `yyyy.MM.dd` would have the consequence that the indices would
 roll daily.
 
-## Upgrading to 5.x
+## Upgrading to 5.6.2
 
-Users should be prepared to re-index when migrating from Elasticsearch 2.x to 5.x. There are a number of template changes, most notably around
+Users should be prepared to re-index when migrating from Elasticsearch 2.3.3 to 5.6.2. There are a number of template changes, most notably around
 string type handling, that may cause issues when upgrading.
 
 [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html)
@@ -248,7 +249,7 @@ Notes on other settings for types in ES
 * [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html)
 * [https://www.elastic.co/blog/strings-are-dead-long-live-strings](https://www.elastic.co/blog/strings-are-dead-long-live-strings)
 
-## Using Metron with Elasticsearch 2.x
+## Using Metron with Elasticsearch 5.6.2
 
 With Elasticsearch 2.x, there is a requirement that all sensors templates have a nested alert field defined.  This field is a dummy field, and will be obsolete in Elasticsearch 5.x.  See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information
 

From 06cf92e44f791634de24d95192d0cc0a5df373de Mon Sep 17 00:00:00 2001
From: cstella 
Date: Tue, 28 Nov 2017 13:48:55 -0500
Subject: [PATCH 54/59] Guids are keywords.

---
 .../METRON/CURRENT/package/files/bro_index.template            | 3 +++
 .../METRON/CURRENT/package/files/snort_index.template          | 3 +++
 .../METRON/CURRENT/package/files/yaf_index.template            | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index b1ed8608a6..01d4f7bae5 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -1306,6 +1306,9 @@
         },
         "content_type": {
           "type": "keyword"
+        },
+        "guid": {
+          "type": "keyword"
         }
       }
     }
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index 0ddb631fd9..43b3ca40f4 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -195,6 +195,9 @@
         "ttl": {
           "type": "integer"
         },
+        "guid": {
+          "type": "keyword"
+        },
         "alert": {
           "type": "nested"
         }
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
index 0f4cd1d990..b6965f9ea7 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -210,6 +210,9 @@
         },
         "alert": {
           "type": "nested"
+        },
+        "guid": {
+          "type": "keyword"
         }
       }
     }

From 0fb19df1045c3b7e81b7cda799b59d31f93e1ae7 Mon Sep 17 00:00:00 2001
From: cstella 
Date: Tue, 28 Nov 2017 13:57:18 -0500
Subject: [PATCH 55/59] Warn level is better.

---
 .../elasticsearch/dao/ElasticsearchColumnMetadataDao.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
index c2e2627ab5..73f0e19f47 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
@@ -114,7 +114,7 @@ public Map getColumnMetadata(List indices) throws IOE
                 FieldType previousType = indexColumnMetadata.get(field);
                 if (!type.equals(previousType)) {
                   String previousIndexName = previousIndices.get(field);
-                  LOG.error(String.format(
+                  LOG.warn(String.format(
                           "Field type mismatch: %s.%s has type %s while %s.%s has type %s.  Defaulting type to %s.",
                           indexName, field, type.getFieldType(),
                           previousIndexName, field, previousType.getFieldType(),

From 20c61399e970671a571db044ea483f49e0805abf Mon Sep 17 00:00:00 2001
From: Michael Miklavcic 
Date: Wed, 29 Nov 2017 17:37:23 -0700
Subject: [PATCH 56/59] Add Metron Error Dashboard. Documentation fixes and
 additions.

---
 metron-deployment/README.md                   | 11 ++++++--
 .../scripts/dashboard/dashboard-bulkload.json | 28 +++++++++++++++++--
 .../metron-elasticsearch/README.md            |  3 ++
 3 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/metron-deployment/README.md b/metron-deployment/README.md
index f3844219fb..8c6b939003 100644
--- a/metron-deployment/README.md
+++ b/metron-deployment/README.md
@@ -184,7 +184,7 @@ You can modify dashboards in Kibana and bring those changes into the core MPack
 2. Convert the data into the ES bulk load format
 3. Replace the dashboard-bulkload.json file in the Kibana MPack.
 
-You can export the .kibana index using a tool like [https://github.com/taskrabbit/elasticsearch-dump] (https://github.com/taskrabbit/elasticsearch-dump). The important
+You can export the .kibana index using a tool like [https://github.com/taskrabbit/elasticsearch-dump](https://github.com/taskrabbit/elasticsearch-dump). The important
 feature is to have one document per line. Here's an exmaple export using elasticsearch-dump
 
 ```
@@ -201,7 +201,14 @@ as follows
 ```
 java -cp $METRON_HOME/lib/metron-elasticsearch-0.4.2-uber.jar org.apache.metron.elasticsearch.bulk.ElasticsearchImportExport \
   ~/dashboard-data.json \
-  ~/dashbaord-bulkload.json
+  ~/dashboard-bulkload.json
+```
+
+Locate the "create" command for setting the default index by searching for "5.6.2". Change "create" to "index" so that it modifies the existing value. It should look similar to line 1 below.
+
+```
+{ "index" : { "_id": "5.6.2", "_type": "config" } }
+{"defaultIndex":"AV-S2e81hKs1cXXnFMqN"}
 ```
 
 Now copy this file to the Kibana MPack, overwriting the existing bulk load file. That should be everything needed to backup the dashboard.
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
index c6087136ca..037f1c63d5 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
@@ -2,8 +2,6 @@
 {"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:ca\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:path_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bro_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bro_timestamp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"capture_password\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:exponent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:issuer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_after\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_before\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:sig_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"client\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"compression_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_uids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"connect_info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_channel:orig_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dgmlen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dhcp_host_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dropped\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dst\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"end_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"end_reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_hash\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"established\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethdst\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethlen\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethsrc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"exception\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failed_sensor_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failure_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_desc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"first_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"helo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"history\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"id\",\"type\":\"conflict\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"conflictDescriptions\":{\"integer\":[\"snort_index_2017.11.06.19\",\"snort_index_2017.11.06.20\",\"snort_index_2017.11.06.21\",\"snort_index_2017.11.06.22\",\"snort_index_2017.11.06.23\",\"snort_index_2017.11.07.00\",\"snort_index_2017.11.07.01\"],\"keyword\":[\"bro_index_2017.11.02.23\",\"bro_index_2017.11.03.00\",\"bro_index_2017.11.03.01\",\"bro_index_2017.11.03.02\",\"bro_index_2017.11.03.03\",\"bro_index_2017.11.03.04\",\"bro_index_2017.11.03.13\",\"bro_index_2017.11.06.19\",\"bro_index_2017.11.06.20\",\"bro_index_2017.11.06.22\",\"bro_index_2017.11.06.23\",\"bro_index_2017.11.07.00\",\"bro_index_2017.11.07.01\"]}},{\"name\":\"iflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"in_reply_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"iplen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_alert\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_orig\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_webmail\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"isn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"issuer_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"kex_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_alert\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_reply\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"lease_time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_orig\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_resp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mailfrom\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"md5\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"method\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"missed_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"missing_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"msg_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"n\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"next_protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"note\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"notice\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"oct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"overflow_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"parent_fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"password\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer_descr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"pkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"port_num\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"proto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message_bytes\",\"type\":\"unknown\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"rcode\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcode_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcptto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"referrer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rejected\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"remote_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"request_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"response_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"result\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resumed\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"riflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"risn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"roct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rpkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ruflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:dns\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:email\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"second_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"seen_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sensor:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sensor:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"service\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha1\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha256\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_generator\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_rev\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"sip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"software_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"source:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src_peer\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"stack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sub\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"suppress_for\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tcpack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpseq\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpwindow\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"threat:triage:level\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:rules:0:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timedout\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tls\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tos\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"total_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ttl\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tunnel_parents\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"unparsed_version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"username\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:major\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor2\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor3\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"x_originating_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"}
 { "create" : { "_id": "AV-Sj0e2hKs1cXXnFMqF", "_type": "visualization" } }
 {"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite.  The default sensor suite includes [\\n                            Snort](https://www.snort.org/), [\\n                            Bro](https://www.bro.org/), and [\\n                            YAF](https://tools.netsa.cert.org/yaf/).  One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry.  In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.'  Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
-{ "create" : { "_id": "AV-Yh94VdXwc6Ua9Muh0", "_type": "index-pattern" } }
-{"title":"error_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"error_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_hash\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"exception\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failed_sensor_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message_bytes\",\"type\":\"unknown\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"stack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"}
 { "index" : { "_id": "5.6.2", "_type": "config" } }
 {"defaultIndex":"AV-S2e81hKs1cXXnFMqN"}
 { "create" : { "_id": "AV-dVurck7f2nZ-iH3Ka", "_type": "visualization" } }
@@ -62,3 +60,29 @@
 {"title":"Web Request Header Overview","visState":"{\"title\":\"Web Request Header Overview\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets.  In this example, Bro is extracting HTTP(S) requests being made over the network.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
 { "create" : { "_id": "AV-dYtN5k7f2nZ-iH3Kd", "_type": "visualization" } }
 {"title":"YAF Overview","visState":"{\"title\":\"YAF Overview\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records.  These flow records provide significant visibility of the actors communicating over the target network.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJrZ63HhN77dHPFvpn", "_type": "visualization" } }
+{"title":"Errors By Error Type","visState":"{\"title\":\"Errors By Error Type\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":false,\"rotate\":0},\"title\":{\"text\":\"error_type: Descending\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true},{\"show\":true,\"mode\":\"stacked\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"data\":{\"id\":\"3\",\"label\":\"Unique Datapoint Count\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\",\"orderBucketsBySum\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Count\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"error_hash\",\"customLabel\":\"Unique Datapoint Count\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"error_type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"colors\":{\"Count\":\"#052B51\",\"Unique Datapoint Count\":\"#1F78C1\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJ-a0-HhN77dHPFv8N", "_type": "visualization" } }
+{"title":"Errors By Source Type","visState":"{\"title\":\"Errors By Source Type\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"rotate\":0},\"title\":{\"text\":\"failed_sensor_type: Descending\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"data\":{\"id\":\"2\",\"label\":\"Count\"},\"valueAxis\":\"ValueAxis-1\"},{\"show\":true,\"mode\":\"stacked\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"data\":{\"id\":\"4\",\"label\":\"Unique Datapoint Count\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Count\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"failed_sensor_type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"2\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"error_hash\",\"customLabel\":\"Unique Datapoint Count\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"colors\":{\"Count\":\"#0A437C\",\"Unique Datapoint Count\":\"#1F78C1\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJQwsZHhN77dHPFvPi", "_type": "visualization" } }
+{"title":"Error Histogram By Sensor Type","visState":"{\"title\":\"Error Histogram By Sensor Type\",\"type\":\"metrics\",\"params\":{\"id\":\"4543cc20-d53a-11e7-895b-2fd1d92b9d92\",\"type\":\"timeseries\",\"series\":[{\"id\":\"4543cc21-d53a-11e7-895b-2fd1d92b9d92\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"4543cc22-d53a-11e7-895b-2fd1d92b9d92\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"terms_field\":\"failed_sensor_type\",\"label\":\"Count\",\"split_filters\":[{\"color\":\"rgba(0,156,224,1)\",\"id\":\"aaaed220-d53b-11e7-895b-2fd1d92b9d92\"}],\"terms_order_by\":\"_count\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"error_index*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"id\":\"7dd75420-d53b-11e7-895b-2fd1d92b9d92\"}],\"gauge_color_rules\":[{\"id\":\"7eadde00-d53b-11e7-895b-2fd1d92b9d92\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"background_color_rules\":[{\"id\":\"811e68d0-d53b-11e7-895b-2fd1d92b9d92\"}]},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJNZY3HhN77dHPFvMT", "_type": "visualization" } }
+{"title":"Unique Error Messages","visState":"{\"title\":\"Unique Error Messages\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"error_hash\",\"customLabel\":\"Unique Error Messages\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAFQPTnHhN77dHPFuOo", "_type": "visualization" } }
+{"title":"Total Error Messages","visState":"{\"title\":\"Total Error Messages\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"fontSize\":60,\"labelColor\":false,\"subText\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total Error Messages\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJiwTHHhN77dHPFvhJ", "_type": "visualization" } }
+{"title":"Unique Error Histogram By Sensor Type","visState":"{\"title\":\"Unique Error Histogram By Sensor Type\",\"type\":\"metrics\",\"params\":{\"id\":\"9009fc50-d546-11e7-9797-db35a5537598\",\"type\":\"timeseries\",\"series\":[{\"id\":\"9009fc51-d546-11e7-9797-db35a5537598\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"9009fc52-d546-11e7-9797-db35a5537598\",\"type\":\"cardinality\",\"field\":\"error_hash\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"terms_field\":\"failed_sensor_type\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"error_index*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJkcsQHhN77dHPFviy", "_type": "visualization" } }
+{"title":"Error Source Proportion","visState":"{\"title\":\"Error Source Proportion\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"failed_sensor_type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"colors\":{\"bro\":\"#0A50A1\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJsGLXHhN77dHPFvqT", "_type": "visualization" } }
+{"title":"Error Type Proportion","visState":"{\"title\":\"Error Type Proportion\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"error_type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"colors\":{\"parser_error\":\"#0A437C\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJ-supHhN77dHPFv8d", "_type": "visualization" } }
+{"title":"Error Hostname Proportion","visState":"{\"title\":\"Error Hostname Proportion\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"hostname\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"colors\":{\"parser_error\":\"#0A437C\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAJ_q99HhN77dHPFv9a", "_type": "visualization" } }
+{"title":"Errors By Hostname","visState":"{\"title\":\"Errors By Hostname\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"rotate\":0},\"title\":{\"text\":\"hostname: Descending\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"data\":{\"id\":\"2\",\"label\":\"Count\"},\"valueAxis\":\"ValueAxis-1\"},{\"show\":true,\"mode\":\"stacked\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"data\":{\"id\":\"4\",\"label\":\"Unique Datapoint Count\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Count\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"hostname\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"2\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"error_hash\",\"customLabel\":\"Unique Datapoint Count\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"colors\":{\"Count\":\"#0A437C\",\"Unique Datapoint Count\":\"#1F78C1\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AV-Yh94VdXwc6Ua9Muh0", "_type": "index-pattern" } }
+{"title":"error_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"error_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_hash\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_type\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"exception\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failed_sensor_type\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"message\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message_bytes\",\"type\":\"unknown\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"stack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"}
+{ "create" : { "_id": "AWAKF7vIHhN77dHPFwBr", "_type": "search" } }
+{"title":"Errors Table","description":"","hits":0,"columns":["failed_sensor_type","error_type","exception","hostname","message","raw_message","error_hash"],"sort":["timestamp","desc"],"version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-Yh94VdXwc6Ua9Muh0\",\"highlightAll\":true,\"version\":true,\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AWAFNXP3HhN77dHPFuOm", "_type": "dashboard" } }
+{"title":"Metron-Error-Dashboard","hits":0,"description":"Metron error dashboard","panelsJSON":"[{\"col\":1,\"id\":\"AWAFQPTnHhN77dHPFuOo\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"AWAJNZY3HhN77dHPFvMT\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWAJQwsZHhN77dHPFvPi\",\"panelIndex\":3,\"row\":4,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWAJiwTHHhN77dHPFvhJ\",\"panelIndex\":4,\"row\":8,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWAJkcsQHhN77dHPFviy\",\"panelIndex\":5,\"row\":12,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AWAJrZ63HhN77dHPFvpn\",\"panelIndex\":6,\"row\":12,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWAJsGLXHhN77dHPFvqT\",\"panelIndex\":7,\"row\":15,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AWAJ-a0-HhN77dHPFv8N\",\"panelIndex\":8,\"row\":15,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWAJ-supHhN77dHPFv8d\",\"panelIndex\":9,\"row\":18,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AWAJ_q99HhN77dHPFv9a\",\"panelIndex\":10,\"row\":18,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":7,\"panelIndex\":11,\"type\":\"search\",\"id\":\"AWAKF7vIHhN77dHPFwBr\",\"col\":1,\"row\":21,\"columns\":[\"failed_sensor_type\",\"error_type\",\"exception\",\"hostname\",\"message\",\"raw_message\",\"error_hash\"],\"sort\":[\"timestamp\",\"desc\"]}]","optionsJSON":"{\"darkTheme\":false}","uiStateJSON":"{\"P-1\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-2\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}}","version":1,"timeRestore":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"}}
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index d235d0935b..6009e48946 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -49,6 +49,9 @@ string type handling, that may cause issues when upgrading.
 
 [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html)
 
+Be aware that if you add a new string value and want to be able to filter and search on this value from the Alerts UI, you **must** add a mapping for that type to
+the appropriate Elasticsearch template. Below is more detail on how to choose the appropriate mapping type for your string value.
+
 ## Type Mappings
 
 Type mappings have changed quite a bit from ES 2.x -> 5.x. Here is a brief rundown of the biggest changes. More detailed references from Elasticsearch

From 0ecbef5fd36132cb8c2a066ee6501dadd74fb519 Mon Sep 17 00:00:00 2001
From: Michael Miklavcic 
Date: Fri, 8 Dec 2017 16:08:44 -0700
Subject: [PATCH 57/59] Fix bug with Netty and Elasticsearch 5 when num
 executors in Storm is increased

---
 .../netty/utils/NettyRuntimeWrapper.java      | 34 +++++++++++++++++++
 metron-platform/metron-elasticsearch/pom.xml  |  6 ----
 .../utils/ElasticsearchUtils.java             | 29 ++++++++--------
 3 files changed, 49 insertions(+), 20 deletions(-)
 create mode 100644 metron-platform/elasticsearch-shaded/src/main/java/org/apache/metron/netty/utils/NettyRuntimeWrapper.java

diff --git a/metron-platform/elasticsearch-shaded/src/main/java/org/apache/metron/netty/utils/NettyRuntimeWrapper.java b/metron-platform/elasticsearch-shaded/src/main/java/org/apache/metron/netty/utils/NettyRuntimeWrapper.java
new file mode 100644
index 0000000000..eda9e6a5d7
--- /dev/null
+++ b/metron-platform/elasticsearch-shaded/src/main/java/org/apache/metron/netty/utils/NettyRuntimeWrapper.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.netty.utils;
+
+import io.netty.util.NettyRuntime;
+
+/**
+ * When working with shaded artifacts that have package relocation, you can't reference the deps
+ * expliticly in some IDE's, e.g. IntelliJ. This allows the shading and relocating to be isolated
+ * to the specific project that contains the dependency.
+ */
+public class NettyRuntimeWrapper {
+
+  public static int availableProcessors() {
+    return NettyRuntime.availableProcessors();
+  }
+
+}
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 3699be9679..7306dbb0fd 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -33,12 +33,6 @@
             elasticsearch-shaded
             ${project.parent.version}
         
-        
         
             org.apache.metron
             metron-enrichment
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
index f948e398a4..4b73b84c1e 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
@@ -17,20 +17,11 @@
  */
 package org.apache.metron.elasticsearch.utils;
 
+import static java.lang.String.format;
+
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
-import org.apache.commons.lang.StringUtils;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.lang.invoke.MethodHandles;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -41,15 +32,19 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import org.apache.commons.lang.StringUtils;
 import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.netty.utils.NettyRuntimeWrapper;
+import org.codehaus.jackson.map.ObjectMapper;
 import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.transport.client.PreBuiltTransportClient;
-import java.util.Optional;
-
-import static java.lang.String.format;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ElasticsearchUtils {
 
@@ -122,6 +117,12 @@ public static TransportClient getClient(Map globalConfiguration,
     Settings settings = settingsBuilder.build();
     TransportClient client;
     try{
+      LOG.info("Number of available processors in Netty: {}", NettyRuntimeWrapper.availableProcessors());
+      // Netty sets available processors statically and if an attempt is made to set it more than
+      // once an IllegalStateException is thrown by NettyRuntime.setAvailableProcessors(NettyRuntime.java:87)
+      // https://discuss.elastic.co/t/getting-availableprocessors-is-already-set-to-1-rejecting-1-illegalstateexception-exception/103082
+      // https://discuss.elastic.co/t/elasticsearch-5-4-1-availableprocessors-is-already-set/88036
+      System.setProperty("es.set.netty.runtime.available.processors", "false");
       client = new PreBuiltTransportClient(settings);
       for(HostnamePort hp : getIps(globalConfiguration)) {
         client.addTransportAddress(

From 3f2c34ddf8bd28dac3601af7b73c7f4f24fb9ac8 Mon Sep 17 00:00:00 2001
From: Michael Miklavcic 
Date: Thu, 4 Jan 2018 08:09:25 -0700
Subject: [PATCH 58/59] Updates per comments from justinleet

---
 metron-platform/metron-elasticsearch/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index dbd9f4efa9..1e150181d3 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -23,7 +23,7 @@ limitations under the License.
 * [Properties](#properties)
 * [Upgrading to 5.6.2](#upgrading-to-562)
 * [Type Mappings](#type-mappings)
-* [Using Metron with Elasticsearch 5.x](#using-metron-with-elasticsearch-5x)
+* [Using Metron with Elasticsearch 5.6.2](#using-metron-with-elasticsearch-562)
 * [Installing Elasticsearch Templates](#installing-elasticsearch-templates)
 
 ## Introduction
@@ -271,7 +271,7 @@ Notes on other settings for types in ES
 
 ## Using Metron with Elasticsearch 5.6.2
 
-With Elasticsearch 2.x, there is a requirement that all sensors templates have a nested alert field defined.  This field is a dummy field, and will be obsolete in Elasticsearch 5.x.  See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information
+There is a requirement that all sensors templates have a nested alert field defined.  This field is a dummy field.  See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information
 
 Without this field, an error will be thrown during ALL searches (including from UIs, resulting in no alerts being found for any sensor). This error will be found in the REST service's logs.
 
@@ -282,7 +282,7 @@ QueryParsingException[[nested] failed to find nested object under path [alert]];
 
 There are two steps to resolve this issue.  First is to update the Elasticsearch template for each sensor, so any new indices have the field. This requires retrieving the template, removing an extraneous JSON field so we can put it back later, and adding our new field.
 
-Make sure to set the ELASTICSEARCH variable appropriately. $SENSOR can contain wildcards, so if rollover has occurred, it's not necessary to do each index individually. The example here appends `index*` to get all indexes for a the provided sensor.
+Make sure to set the ELASTICSEARCH variable appropriately. $SENSOR can contain wildcards, so if rollover has occurred, it's not necessary to do each index individually. The example here appends `index*` to get all indexes for the provided sensor.
 
 ```
 export ELASTICSEARCH="node1"

From 04a9292f54d4eebe52a1a54a947e60bee8704b5d Mon Sep 17 00:00:00 2001
From: Michael Miklavcic 
Date: Fri, 5 Jan 2018 15:50:56 -0700
Subject: [PATCH 59/59] Fix e2e tests with ES 5.6.2 templates

---
 .../mock-data/alerts_ui_e2e_index.template    | 827 +++++++++++++-----
 .../metron-alerts/e2e/utils/e2e_util.ts       |  32 +-
 2 files changed, 610 insertions(+), 249 deletions(-)

diff --git a/metron-interface/metron-alerts/e2e/mock-data/alerts_ui_e2e_index.template b/metron-interface/metron-alerts/e2e/mock-data/alerts_ui_e2e_index.template
index 773759286e..caf593c987 100644
--- a/metron-interface/metron-alerts/e2e/mock-data/alerts_ui_e2e_index.template
+++ b/metron-interface/metron-alerts/e2e/mock-data/alerts_ui_e2e_index.template
@@ -2,9 +2,6 @@
   "template": "alerts_ui_e2e_index",
   "mappings": {
     "alerts_ui_e2e_doc": {
-      "_timestamp": {
-        "enabled": true
-      },
       "dynamic_templates": [
       {
         "geo_location_point": {
@@ -20,8 +17,7 @@
           "match": "enrichments:geo:*:country",
           "match_mapping_type": "*",
           "mapping": {
-            "type": "string",
-            "index": "not_analyzed"
+            "type": "keyword"
           }
         }
       },
@@ -30,8 +26,7 @@
           "match": "enrichments:geo:*:city",
           "match_mapping_type": "*",
           "mapping": {
-            "type": "string",
-            "index": "not_analyzed"
+            "type": "keyword"
           }
         }
       },
@@ -40,8 +35,7 @@
           "match": "enrichments:geo:*:locID",
           "match_mapping_type": "*",
           "mapping": {
-            "type": "string",
-            "index": "not_analyzed"
+            "type": "keyword"
           }
         }
       },
@@ -50,8 +44,7 @@
           "match": "enrichments:geo:*:dmaCode",
           "match_mapping_type": "*",
           "mapping": {
-            "type": "string",
-            "index": "not_analyzed"
+            "type": "keyword"
           }
         }
       },
@@ -60,8 +53,7 @@
           "match": "enrichments:geo:*:postalCode",
           "match_mapping_type": "*",
           "mapping": {
-            "type": "string",
-            "index": "not_analyzed"
+            "type": "keyword"
           }
         }
       },
@@ -98,25 +90,27 @@
           "mapping": {
             "type": "float"
           },
-          "match": "threat.triage.rules:*:score",
+          "match": "threat:triage:*score",
           "match_mapping_type": "*"
         }
       },
       {
         "threat_triage_reason": {
           "mapping": {
-            "type": "string"
+            "type": "text",
+            "fielddata": "true"
           },
-          "match": "threat.triage.rules:*:reason",
+          "match": "threat:triage:rules:*:reason",
           "match_mapping_type": "*"
         }
       },
       {
         "threat_triage_name": {
           "mapping": {
-            "type": "string"
+            "type": "text",
+            "fielddata": "true"
           },
-          "match": "threat.triage.rules:*:name",
+          "match": "threat:triage:rules:*:name",
           "match_mapping_type": "*"
         }
       }
@@ -131,14 +125,13 @@
          * Be careful when modifying this file to not unintentionally affect other logs.
          * For instance, the "version" field exists in the HTTP, SSL, and SSH logs.  If you
          * were to only consider the SSH log, you would set the type to integer, but because
-         * in the SSL and HTTP logs version is a string, we must set the type to string.
+         * in the SSL and HTTP logs version is a string, we must set the type to keyword.
          */
         /*
          * Metron-specific fields
          */
         "source:type": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         /*
          * Widely-used Bro fields (potentially renamed during Metron ingest)
@@ -148,8 +141,7 @@
           "format": "epoch_millis"
         },
         "uid": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "alert": {
           "type": "nested"
@@ -171,6 +163,12 @@
          * https://www.bro.org/sphinx/scripts/base/protocols/http/main.bro.html#type-HTTP::Info
          *
          * Notable Fields
+         *   Field:     method
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     uri
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
          *   Field:     password
          *   Notes:     Field exists in the HTTP and FTP logs
          *
@@ -178,46 +176,54 @@
          *   Notes:     Field exists in the HTTP and FTP logs
          *
          *   Field:     trans_depth
-         *   Notes:     Field exists in the HTTP and SMTP logs
+         *   Notes:     Field exists in the HTTP, SMTP, and SIP logs
          *
          *   Field:     user_agent
-         *   Notes:     Field exists in the HTTP and SMTP logs
+         *   Notes:     Field exists in the HTTP, SMTP, and SIP logs
          *
          *   Field:     version
          *   Notes:     Field exists in the HTTP, SSL, and SSH logs
          *
          *   Field:     host
-         *   Notes:     Field exists in the HTTP and Software logs
+         *   Notes:     Field exists in the HTTP, KnownCerts, and Software logs
          *
          *   Field:     username
          *   Notes:     Field exists in the HTTP and RADIUS logs
+         *
+         *   Field:     status_code
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     status_msg
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     request_body_len
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     response_body_len
+         *   Notes:     Field exists in the HTTP and SIP logs
          */
         "trans_depth": {
           "type": "integer"
         },
         "method": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "host": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "uri": {
-          "type": "string",
-          "index": "not_analyzed",
+          "type": "keyword",
           "ignore_above": 8191
         },
         "referrer": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "version": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "user_agent": {
-          "type": "string"
+          "type": "text",
+          "fielddata": "true"
         },
         "request_body_len": {
           "type": "long"
@@ -229,19 +235,46 @@
           "type": "integer"
         },
         "status_msg": {
+          "type": "keyword"
+        },
+        "info_code": {
+          "type": "integer"
+        },
+        "info_msg": {
           "type": "string",
           "index": "not_analyzed"
         },
-        "username": {
+        "tags": {
           "type": "string",
           "index": "not_analyzed"
         },
+        "username": {
+          "type": "keyword"
+        },
         "password": {
+          "type": "keyword"
+        },
+        "proxied": {
           "type": "string",
           "index": "not_analyzed"
         },
-        "capture_password": {
-          "type": "boolean"
+        "orig_fuids": {
+          "type": "string"
+        },
+        "orig_filenames": {
+          "type": "string"
+        },
+        "orig_mime_types": {
+          "type": "string"
+        },
+        "resp_fuids": {
+          "type": "string"
+        },
+        "resp_filenames": {
+          "type": "string"
+        },
+        "resp_mime_types": {
+          "type": "string"
         },
         /*
          * DNS log support
@@ -253,38 +286,41 @@
          *
          *   Field:     trans_id
          *   Notes:     Field exists in the DNS and DHCP logs
+         *
+         *   Field:     rtt
+         *   Notes:     This field uses the "interval" type, which may need handled differently.
+         *              https://www.bro.org/sphinx-git/script-reference/types.html#type-interval
          */
         "proto": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "trans_id": {
           "type": "long"
         },
-        "query": {
+        "rtt": {
           "type": "string",
           "index": "not_analyzed"
         },
+        "query": {
+          "type": "keyword"
+        },
         "qclass": {
           "type": "integer"
         },
         "qclass_name": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "qtype": {
           "type": "integer"
         },
         "qtype_name": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "rcode": {
           "type": "integer"
         },
         "rcode_name": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "AA": {
           "type": "boolean"
@@ -302,6 +338,10 @@
           "type": "integer"
         },
         "answers": {
+          "type": "text",
+          "fielddata": "true"
+        },
+        "TTLs": {
           "type": "string"
         },
         "rejected": {
@@ -322,58 +362,46 @@
          *   Notes:     Field exists in the Conn and Files logs
          */
         "service": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "duration": {
           "type": "float"
         },
         "orig_bytes": {
-          "type": "long",
-          "index": "not_analyzed"
+          "type": "long"
         },
         "resp_bytes": {
-          "type": "long",
-          "index": "not_analyzed"
+          "type": "long"
         },
         "conn_state": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "local_orig": {
           "type": "boolean"
         },
         "local_resp": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "missed_bytes": {
-          "type": "long",
-          "index": "not_analyzed"
+          "type": "long"
         },
         "history": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "orig_pkts": {
-          "type": "long",
-          "index": "not_analyzed"
+          "type": "long"
         },
         "orig_ip_bytes": {
-          "type": "long",
-          "index": "not_analyzed"
+          "type": "long"
         },
         "resp_pkts": {
-          "type": "long",
-          "index": "not_analyzed"
+          "type": "long"
         },
         "resp_ip_bytes": {
-          "type": "long",
-          "index": "not_analyzed"
+          "type": "long"
         },
         "tunnel_parents": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         /*
          * DPD log support
@@ -384,12 +412,10 @@
          *   Notes:     Field exists in the DNS, Conn, DPD, and Notice logs
          */
         "analyzer": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "failure_reason": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         /*
          * FTP log support
@@ -406,22 +432,22 @@
          *   Notes:     Field exists in the FTP and Files logs
          *
          *   Field:     fuid
-         *   Notes:     Field exists in the FTP and Notice logs
+         *   Notes:     Field exists in the FTP, Files, and Notice logs
          */
         "user": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "command": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "arg": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "mime_type": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "file_size": {
@@ -431,8 +457,7 @@
           "type": "integer"
         },
         "reply_msg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "data_channel:passive": {
           "type": "boolean"
@@ -447,15 +472,15 @@
           "type": "integer"
         },
         "cwd": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "passive": {
           "type": "boolean"
         },
         "fuid": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         /*
          * Files log support
@@ -470,25 +495,34 @@
          *
          *   Field:     mime_type
          *   Notes:     Field exists in the FTP and Files logs
+         *
+         *   Field:     duration
+         *   Notes:     Field exists in the Conn and Files logs
+         *
+         *   Field:     local_orig
+         *   Notes:     Field exists in the Conn and Files logs
+         *
+         *   Field:     fuid
+         *   Notes:     Field exists in the FTP, Files, and Notice logs
          */
         "conn_uids": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "source": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "depth": {
           "type": "integer"
         },
         "analyzers": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "filename": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "is_orig": {
           "type": "boolean"
@@ -509,113 +543,143 @@
           "type": "boolean"
         },
         "parent_fuid": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "md5": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "sha1": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "sha256": {
+          "type": "keyword"
+        },
+        "extracted": {
           "type": "string",
           "index": "not_analyzed"
         },
+        "extracted_cutoff": {
+          "type": "boolean"
+        },
+        "extracted_size": {
+          "type": "long"
+        },
         /*
          * Known::CertInfo log support
          * https://www.bro.org/sphinx/scripts/policy/protocols/ssl/known-certs.bro.html#type-Known::CertsInfo
          *
          * Notable Fields
+         *   Field:     host
+         *   Notes:     Field exists in the HTTP, KnownCerts, and Software logs
+         *
          *   Field:     subject
-         *   Notes:     Field exists in the Known::CertInfo and SMTP logs
+         *   Notes:     Field exists in the KnownCerts, SMTP, SIP, and SSL logs
          */
         "port_num": {
           "type": "integer"
         },
         "subject": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "issuer_subject": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "serial": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         /*
          * SMTP log support
          * https://www.bro.org/sphinx/scripts/base/protocols/smtp/main.bro.html#type-SMTP::Info
          *
          * Notable Fields
+         *   Field:     trans_depth
+         *   Notes:     Field exists in the HTTP, SMTP, and SIP logs
+         *
+         *   Field:     date
+         *   Notes:     Field exists in the SMTP and SIP logs
+         *
          *   Field:     subject
-         *   Notes:     Field exists in the Known::CertInfo and SMTP logs
+         *   Notes:     Field exists in the KnownCerts, SMTP, SIP, and SSL logs
+         *
+         *   Field:     reply_to
+         *   Notes:     Field exists in the SMTP and SIP logs
+         *
+         *   Field:     user_agent
+         *   Notes:     Field exists in the HTTP, SMTP, and SIP logs
          */
         "helo": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "mailfrom": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "rcptto": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "date": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "from": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "to": {
+          "type": "text",
+          "fielddata": "true",
+          "analyzer": "simple"
+        },
+        "cc": {
           "type": "string",
           "analyzer": "simple"
         },
         "reply_to": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "msg_id": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "in_reply_to": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "x_originating_ip": {
           "type": "ip"
         },
         "first_received": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "second_received": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "last_reply": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         "path": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "tls": {
           "type": "boolean"
         },
         "fuids": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "is_webmail": {
           "type": "boolean"
@@ -627,51 +691,82 @@
          * Notable Fields
          *   Field:     version
          *   Notes:     Field exists in the HTTP, SSL, and SSH logs
+         *
+         *   Field:     subject
+         *   Notes:     Field exists in the KnownCerts, SMTP, SIP, and SSL logs
          */
         "cipher": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "curve": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "server_name": {
+          "type": "keyword"
+        },
+        "resumed": {
+          "type": "boolean"
+        },
+        "server_appdata": {
           "type": "string",
           "index": "not_analyzed"
         },
-        "resumed": {
+        "client_appdata": {
           "type": "boolean"
         },
         "last_alert": {
+          "type": "keyword"
+        },
+        "next_protocol": {
+          "type": "keyword"
+        },
+        "established": {
+          "type": "boolean"
+        },
+        "cert_chain_fuids": {
+          "type": "string"
+        },
+        "client_cert_chain_fuids": {
+          "type": "string"
+        },
+        "issuer": {
           "type": "string",
           "index": "not_analyzed"
         },
-        "next_protocol": {
+        "client_subject": {
           "type": "string",
           "index": "not_analyzed"
         },
-        "established": {
-          "type": "boolean"
+        "client_issuer": {
+          "type": "string",
+          "index": "not_analyzed"
+        },
+        "validation_status": {
+          "type": "string",
+          "index": "not_analyzed"
         },
         /*
          * Weird log support
          * https://www.bro.org/sphinx/scripts/base/frameworks/notice/weird.bro.html#type-Weird::Info
+         *
+         * Notable Fields
+         *   Field:     peer
+         *   Notes:     Field exists in the Weird, CaptureLoss, and Stats logs
+         *
+         *   Field:     name
+         *   Notes:     Field exists in the Weird and LoadedScripts logs
          */
         "name": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "addl": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "notice": {
           "type": "boolean"
         },
         "peer": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         /*
          * Notice log support
@@ -679,30 +774,40 @@
          *
          * Notable Fields
          *   Field:     fuid
-         *   Notes:     Field exists in the FTP and Notice logs
+         *   Notes:     Field exists in the FTP, Files, and Notice logs
          *
          *   Field:     proto
          *   Notes:     Field exists in the DNS, Conn, DPD, and Notice logs
+         *
+         *   Field:     remote_location:country_code
+         *   Notes:     Field exists in the Notice and SSH logs
+         *
+         *   Field:     remote_location:region
+         *   Notes:     Field exists in the Notice and SSH logs
+         *
+         *   Field:     remote_location:city
+         *   Notes:     Field exists in the Notice and SSH logs
+         *
+         *   Field:     remote_location:latitude
+         *   Notes:     Field exists in the Notice and SSH logs
+         *
+         *   Field:     remote_location:longitude
+         *   Notes:     Field exists in the Notice and SSH logs
          */
         "file_mime_type": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "file_desc": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "note": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "msg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "sub": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "src": {
           "type": "ip"
@@ -711,52 +816,60 @@
           "type": "ip"
         },
         "p": {
-          "type": "integer",
-          "index": "not_analyzed"
+          "type": "integer"
         },
         "n": {
-          "type": "integer",
-          "index": "not_analyzed"
+          "type": "integer"
         },
         "src_peer": {
           "type": "ip"
         },
         "peer_descr": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "actions": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "suppress_for": {
-          "type": "double",
-          "index": "not_analyzed"
+          "type": "double"
         },
         "dropped": {
           "type": "boolean"
         },
+        "remote_location:country_code": {
+          "type": "string"
+        },
+        "remote_location:region": {
+          "type": "string"
+        },
+        "remote_location:city": {
+          "type": "string"
+        },
+        "remote_location:latitude": {
+          "type": "double"
+        },
+        "remote_location:longitude": {
+          "type": "double"
+        },
         /*
          * DHCP log support
          * https://www.bro.org/sphinx/scripts/base/protocols/dhcp/main.bro.html#type-DHCP::Info
          *
          * Notable Fields
+         *   Field:     mac
+         *   Notes:     Field exists in the DHCP, RADIUS, and KnownDevices logs
+         *
          *   Field:     trans_id
          *   Notes:     Field exists in the DNS and DHCP logs
-         *
-         *   Field:     mac
-         *   Notes:     Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs
          */
         "mac": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "assigned_ip": {
           "type": "ip"
         },
         "lease_time": {
-          "type": "float",
-          "index": "not_analyzed"
+          "type": "float"
         },
         /*
          * SSH log support
@@ -765,49 +878,54 @@
          * Notable Fields
          *   Field:     version
          *   Notes:     Field exists in the HTTP, SSL, and SSH logs
+         *
+         *   Field:     remote_location:country_code
+         *   Notes:     Field exists in the Notice and SSH logs
+         *
+         *   Field:     remote_location:region
+         *   Notes:     Field exists in the Notice and SSH logs
+         *
+         *   Field:     remote_location:city
+         *   Notes:     Field exists in the Notice and SSH logs
+         *
+         *   Field:     remote_location:latitude
+         *   Notes:     Field exists in the Notice and SSH logs
+         *
+         *   Field:     remote_location:longitude
+         *   Notes:     Field exists in the Notice and SSH logs
          */
         "auth_success": {
           "type": "boolean"
         },
         "auth_attempts": {
-          "type": "integer",
-          "index": "not_analyzed"
+          "type": "integer"
         },
         "direction": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "client": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "server": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "cipher_alg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "mac_alg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "compression_alg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "kex_alg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "host_key_alg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "host_key": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         /*
          * Software log support
@@ -815,38 +933,32 @@
          *
          * Notable Fields
          *   Field:     host
-         *   Notes:     Field exists in the HTTP and Software logs
+         *   Notes:     Field exists in the HTTP, KnownCerts, and Software logs
          */
         "host_p": {
-          "type": "integer",
-          "index": "not_analyzed"
+          "type": "integer"
         },
         "software_type": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "version:major": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "version:minor": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "version:minor2": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "version:minor3": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "version:addl": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "unparsed_version": {
-          "type": "string",
+          "type": "text",
+          "fielddata": "true",
           "analyzer": "simple"
         },
         /*
@@ -858,16 +970,29 @@
          *   Notes:     Field exists in the HTTP and RADIUS logs
          *
          *   Field:     mac
-         *   Notes:     Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs
+         *   Notes:     Field exists in the DHCP, RADIUS, and KnownDevices logs
+         *
+         *   Field:     ttl
+         *   Notes:     This field uses the "interval" type, which may need handled differently.
+         *              https://www.bro.org/sphinx-git/script-reference/types.html#type-interval
          */
+        "framed_addr": {
+          "type": "ip"
+        },
         "remote_ip": {
           "type": "ip"
         },
         "connect_info": {
+          "type": "keyword"
+        },
+        "reply_msg": {
           "type": "string",
           "index": "not_analyzed"
         },
         "result": {
+          "type": "keyword"
+        },
+        "ttl": {
           "type": "string",
           "index": "not_analyzed"
         },
@@ -880,82 +1005,64 @@
          *   Notes:     In other bro records, the id field is of type conn_id, so it is
          *              expanded before being logged into 4 fields, all of which are addressed
          *              under the "Widely-used Bro fields" section of this template.  In X509
-         *              logs, however, id is a string to identify the certificate file id.
+         *              logs, however, id is a keyword to identify the certificate file id.
          */
         "id": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:version": {
-          "type": "integer",
-          "index": "not_analyzed"
+          "type": "integer"
         },
         "certificate:serial": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:subject": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:issuer": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:not_valid_before": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:not_valid_after": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:key_alg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:sig_alg": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:key_type": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:key_length": {
-          "type": "integer",
-          "index": "not_analyzed"
+          "type": "integer"
         },
         "certificate:exponent": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "certificate:curve": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "san:dns": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "san:uri": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "san:email": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "san:ip": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "basic_constraints:ca": {
           "type": "boolean"
         },
         "basic_constraints:path_len": {
-          "type": "integer",
-          "index": "not_analyzed"
+          "type": "integer"
         },
         /*
          * Known::DevicesInfo log support
@@ -963,11 +1070,245 @@
          *
          * Notable Fields
          *   Field:     mac
-         *   Notes:     Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs
+         *   Notes:     Field exists in the DHCP, RADIUS, and KnownDevices logs
          */
         "dhcp_host_name": {
-          "type": "string",
+          "type": "keyword"
+        },
+        /*
+         * RFB::Info log support
+         * https://www.bro.org/sphinx-git/scripts/base/protocols/rfb/main.bro.html#type-RFB::Info
+         */
+        "client_major_version": {
+          "type": "keyword"
+        },
+        "client_minor_version": {
+          "type": "keyword"
+        },
+        "server_major_version": {
+          "type": "keyword"
+        },
+        "server_minor_version": {
+          "type": "keyword"
+        },
+        "authentication_method": {
+          "type": "keyword"
+        },
+        "auth": {
+          "type": "boolean"
+        },
+        "share_flag": {
+          "type": "boolean"
+        },
+        "desktop_name": {
+          "type": "keyword"
+        },
+        "width": {
+          "type": "integer"
+        },
+        "height": {
+          "type": "integer"
+        },
+        /*
+         * Stats::Info log support
+         * https://www.bro.org/sphinx/scripts/policy/misc/stats.bro.html#type-Stats::Info
+         *
+         * Notable Fields
+         *   Field:     peer
+         *   Notes:     Field exists in the Weird, CaptureLoss, and Stats logs
+         *
+         *   Field:     pkt_lag
+         *   Notes:     This field uses the "interval" type, which may need handled differently.
+         *              https://www.bro.org/sphinx-git/script-reference/types.html#type-interval
+         */
+        "mem": {
+          "type": "integer"
+        },
+        "pkts_proc": {
+          "type": "integer"
+        },
+        "bytes_recv": {
+          "type": "integer"
+        },
+        "pkts_dropped": {
+          "type": "integer"
+        },
+        "pkts_link": {
+          "type": "integer"
+        },
+        "pkt_lag": {
+          "type": "keyword"
+        },
+        "events_proc": {
+          "type": "integer"
+        },
+        "events_queued": {
+          "type": "integer"
+        },
+        "active_tcp_conns": {
+          "type": "integer"
+        },
+        "active_udp_conns": {
+          "type": "integer"
+        },
+        "active_icmp_conns": {
+          "type": "integer"
+        },
+        "tcp_conns": {
+          "type": "integer"
+        },
+        "udp_conns": {
+          "type": "integer"
+        },
+        "icmp_conns": {
+          "type": "integer"
+        },
+        "timers": {
+          "type": "integer"
+        },
+        "active_timers": {
+          "type": "integer"
+        },
+        "files": {
+          "type": "integer"
+        },
+        "active_files": {
+          "type": "integer"
+        },
+        "dns_requests": {
+          "type": "integer"
+        },
+        "active_dns_requests": {
+          "type": "integer"
+        },
+        "reassem_tcp_size": {
+          "type": "integer"
+        },
+        "reassem_file_size": {
+          "type": "integer"
+        },
+        "reassem_frag_size": {
+          "type": "integer"
+        },
+        "reassem_unknown_size": {
+          "type": "integer"
+        },
+        /*
+         * CaptureLoss::Info log support
+         * https://www.bro.org/sphinx/scripts/policy/misc/capture-loss.bro.html#type-CaptureLoss::Info
+         *
+         * Notable Fields
+         *   Field:     ts_delta
+         *   Notes:     This field uses the "interval" type, which may need handled differently.
+         *              https://www.bro.org/sphinx-git/script-reference/types.html#type-interval
+         *
+         *   Field:     peer
+         *   Notes:     Field exists in the Weird, CaptureLoss, and Stats logs
+         */
+        "ts_delta": {
+          "type": "keyword"
+        },
+        "gaps": {
+          "type": "integer",
           "index": "not_analyzed"
+        },
+        "acks": {
+          "type": "integer",
+          "index": "not_analyzed"
+        },
+        "percent_lost": {
+          "type": "double",
+          "index": "not_analyzed"
+        },
+        /*
+         * Reporter::Info log support
+         * https://www.bro.org/sphinx/scripts/base/frameworks/reporter/main.bro.html#type-Reporter::Info
+         */
+        "level": {
+          "type": "text",
+          "fielddata": "true",
+          "analyzer": "simple"
+        },
+        "message": {
+          "type": "keyword"
+        },
+        "location": {
+          "type": "keyword"
+        },
+        /*
+         * SIP::Info log support
+         * https://www.bro.org/sphinx/scripts/base/protocols/sip/main.bro.html#type-SIP::Info
+         *
+         * Notable Fields
+         *   Field:     trans_depth
+         *   Notes:     Field exists in the HTTP, SMTP, and SIP logs
+         *
+         *   Field:     method
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     uri
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     date
+         *   Notes:     Field exists in the SMTP and SIP logs
+         *
+         *   Field:     reply_to
+         *   Notes:     Field exists in the SMTP and SIP logs
+         *
+         *   Field:     subject
+         *   Notes:     Field exists in the KnownCerts, SMTP, SIP, and SSL logs
+         *
+         *   Field:     user_agent
+         *   Notes:     Field exists in the HTTP, SMTP, and SIP logs
+         *
+         *   Field:     status_code
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     status_msg
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     request_body_len
+         *   Notes:     Field exists in the HTTP and SIP logs
+         *
+         *   Field:     response_body_len
+         *   Notes:     Field exists in the HTTP and SIP logs
+         */
+        "request_from": {
+          "type": "keyword"
+        },
+        "request_to": {
+          "type": "keyword"
+        },
+        "response_from": {
+          "type": "keyword"
+        },
+        "response_to": {
+          "type": "keyword"
+        },
+        "call_id": {
+          "type": "keyword"
+        },
+        "seq": {
+          "type": "keyword"
+        },
+        "request_path": {
+          "type": "text",
+          "fielddata": "true",
+          "analyzer": "simple"
+        },
+        "response_path": {
+          "type": "text",
+          "fielddata": "true",
+          "analyzer": "simple"
+        },
+        "warning": {
+          "type": "keyword"
+        },
+        "content_type": {
+          "type": "keyword"
+        },
+        "guid": {
+          "type": "keyword"
         }
       }
     }
diff --git a/metron-interface/metron-alerts/e2e/utils/e2e_util.ts b/metron-interface/metron-alerts/e2e/utils/e2e_util.ts
index 92476a4e04..8ae1de1c3d 100644
--- a/metron-interface/metron-alerts/e2e/utils/e2e_util.ts
+++ b/metron-interface/metron-alerts/e2e/utils/e2e_util.ts
@@ -48,10 +48,23 @@ export function waitForStalenessOf (_element ) {
 export function loadTestData() {
   deleteTestData();
 
-  fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.template')
-    .pipe(request.post('http://node1:9200/_template/alerts_ui_e2e_index'));
-  fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.data')
-    .pipe(request.post('http://node1:9200/alerts_ui_e2e_index/alerts_ui_e2e_doc/_bulk'));
+  let template = fs.readFileSync('e2e/mock-data/alerts_ui_e2e_index.template', 'utf8');
+  request({
+    url: 'http://node1:9200/_template/alerts_ui_e2e_index',
+    method: 'POST',
+    body: template
+  }, function(error, response, body) {
+    // add logging if desired
+  });
+
+  let data = fs.readFileSync('e2e/mock-data/alerts_ui_e2e_index.data', 'utf8');
+  request({
+    url: 'http://node1:9200/alerts_ui_e2e_index/alerts_ui_e2e_doc/_bulk',
+    method: 'POST',
+    body: data
+  }, function(error, response, body) {
+    // add logging if desired
+  });
 }
 
 export function deleteTestData() {
@@ -60,8 +73,15 @@ export function deleteTestData() {
 
 export function createMetaAlertsIndex() {
   deleteMetaAlertsIndex();
-  fs.createReadStream('./../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template')
-  .pipe(request.post('http://node1:9200/metaalert_index'));
+
+  let template = fs.readFileSync('./../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template', 'utf8');
+  request({
+    url: 'http://node1:9200/_template/metaalert_index',
+    method: 'POST',
+    body: template
+  }, function(error, response, body) {
+    // add logging if desired
+  });
 }
 
 export function deleteMetaAlertsIndex() {