diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml index 0a26b73f42..f97b97ab98 100644 --- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml +++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml @@ -127,8 +127,6 @@ bolts: configMethods: - name: "withMessageWriter" args: [ref: "kafkaWriter"] - - name: "withMessageGetter" - args: ["NAMED"] streams: diff --git a/metron-deployment/packaging/ambari/metron-mpack/README.md b/metron-deployment/packaging/ambari/metron-mpack/README.md new file mode 100644 index 0000000000..08f28b015d --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/README.md @@ -0,0 +1,20 @@ +# Overview + +Contains the definitions of the various services for the Ambari Management Pack. + +More info can be found at https://cwiki.apache.org/confluence/display/AMBARI/Management+Packs + +## Kibana Dashboards + +The dashboards installed by the Kibana custom action are managed by the dashboard.p file. This file is created by exporting existing dashboards from a running Kibana instance. + +To create a new version of the file, make any necessary changes to Kibana (e.g. on quick-dev), and export with the appropriate command. +``` +python src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py $KIBANA_URL 9200 dashboard.p -s +mv dashboard.p src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p +``` + +Build the Ambari Mpack to get the dashboard updated appropriately. + +Once the MPack is installed, run the Kibana service's action "Load Template" to install dashboards. This will completely overwrite the .kibana in Elasticsearch, so use with caution. + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json index e08f401c86..0b04f128fa 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json @@ -4,8 +4,9 @@ "general_deps" : { "_comment" : "dependencies for all cases", "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"], + "METRON_ENRICHMENT-INSTALL" : ["METRON_INDEXING-INSTALL"], "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"], - "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], + "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START", "METRON_INDEXING-START"], "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"], "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"], "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p old mode 100755 new mode 100644 index 8327eb8ad9..ed5b2bdacd --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p @@ -5,310 +5,310 @@ p3 F1 sV_type p4 -Vvisualization +Vindex-pattern p5 sV_id p6 -VWeb-Request-Type +Vbro* p7 sV_source p8 (dp9 -VvisState +Vfields p10 -V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] p11 -sVdescription +sVtimeFieldName p12 -V -sVtitle +Vtimestamp p13 -VWeb Request Type +sVtitle p14 -sVuiStateJSON +Vbro* p15 -V{} +ssV_index p16 -sVversion +V.kibana p17 -I1 -sVsavedSearchId -p18 -Vweb-search +sa(dp18 +V_score p19 -sVkibanaSavedObjectMeta +F1 +sV_type p20 -(dp21 -VsearchSourceJSON +Vsearch +p21 +sV_id p22 -V{"filter":[]} +Vsnort-search p23 -sssV_index +sV_source p24 -V.kibana -p25 -sa(dp26 -V_score -p27 -F1 -sV_type +(dp25 +Vsort +p26 +(lp27 +Vtimestamp p28 -Vvisualization +aVdesc p29 -sV_id +asVhits p30 -VTop-Snort-Alerts-by-Source +I0 +sVdescription p31 -sV_source +V +sVtitle p32 -(dp33 -VvisState +VSnort Alerts +p33 +sVversion p34 -V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}} +I1 +sVkibanaSavedObjectMeta p35 -sVdescription -p36 -V -sVtitle +(dp36 +VsearchSourceJSON p37 -VTop Snort Alerts by Source +V{"index":"snort*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} p38 -sVuiStateJSON +ssVcolumns p39 -V{} -p40 -sVversion +(lp40 +Vmsg p41 -I1 -sVkibanaSavedObjectMeta +aVsig_id p42 -(dp43 -VsearchSourceJSON +aVip_src_addr +p43 +aVip_src_port p44 -V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +aVip_dst_addr p45 -sssV_index +aVip_dst_port p46 -V.kibana +assV_index p47 -sa(dp48 +V.kibana +p48 +sa(dp49 V_score -p49 +p50 F1 sV_type -p50 -Vvisualization p51 -sV_id +Vsearch p52 -VWelcome +sV_id p53 -sV_source +Vyaf-search p54 -(dp55 -VvisState -p56 -V{"title":"Welcome to Apache Metron","type":"markdown","params":{"markdown":"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\u005cn\u005cnApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\u005cn\u005cnThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron."},"aggs":[],"listeners":{}} +sV_source +p55 +(dp56 +Vsort p57 -sVdescription -p58 -V -sVtitle +(lp58 +Vtimestamp p59 -VWelcome to Apache Metron +aVdesc p60 -sVuiStateJSON +asVhits p61 -V{} +I0 +sVdescription p62 -sVversion +V +sVtitle p63 +VYAF +p64 +sVversion +p65 I1 sVkibanaSavedObjectMeta -p64 -(dp65 -VsearchSourceJSON p66 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p67 -sssV_index +(dp67 +VsearchSourceJSON p68 -V.kibana +V{"index":"yaf*","filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647},"query":{"query_string":{"query":"*","analyze_wildcard":true}}} p69 -sa(dp70 -V_score -p71 -F1 -sV_type +ssVcolumns +p70 +(lp71 +Vip_src_addr p72 -Vsearch +aVip_src_port p73 -sV_id +aVip_dst_addr p74 -Vsnort-search +aVip_dst_port p75 -sV_source +aVprotocol p76 -(dp77 -Vsort +aVduration +p77 +aVpkt p78 -(lp79 -Vtimestamp +assV_index +p79 +V.kibana p80 -aVdesc -p81 -asVhits +sa(dp81 +V_score p82 -I0 -sVdescription +F1 +sV_type p83 -V -sVtitle +Vvisualization p84 -VSnort Alerts +sV_id p85 -sVversion +VWelcome p86 -I1 -sVkibanaSavedObjectMeta +sV_source p87 (dp88 -VsearchSourceJSON +VvisState p89 -V{"index":"snort*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} +V{"title":"Welcome to Apache Metron","type":"markdown","params":{"markdown":"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\u005cn\u005cnApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\u005cn\u005cnThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron."},"aggs":[],"listeners":{}} p90 -ssVcolumns +sVdescription p91 -(lp92 -Vmsg +V +sVtitle +p92 +VWelcome to Apache Metron p93 -aVsig_id +sVuiStateJSON p94 -aVip_src_addr +V{} p95 -aVip_src_port +sVversion p96 -aVip_dst_addr +I1 +sVkibanaSavedObjectMeta p97 -aVip_dst_port -p98 -assV_index +(dp98 +VsearchSourceJSON p99 -V.kibana +V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} p100 -sa(dp101 -V_score +sssV_index +p101 +V.kibana p102 +sa(dp103 +V_score +p104 F1 sV_type -p103 -Vsearch -p104 -sV_id p105 -Vyaf-search +Vvisualization p106 -sV_source +sV_id p107 -(dp108 -Vsort +VTop-Snort-Alerts-by-Source +p108 +sV_source p109 -(lp110 -Vtimestamp +(dp110 +VvisState p111 -aVdesc +V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}} p112 -asVhits -p113 -I0 sVdescription -p114 +p113 V sVtitle +p114 +VTop Snort Alerts by Source p115 -VYAF +sVuiStateJSON p116 -sVversion +V{} p117 +sVversion +p118 I1 sVkibanaSavedObjectMeta -p118 -(dp119 +p119 +(dp120 VsearchSourceJSON -p120 -V{"index":"yaf*","filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647},"query":{"query_string":{"query":"*","analyze_wildcard":true}}} p121 -ssVcolumns +V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p122 -(lp123 -Vip_src_addr +sssV_index +p123 +V.kibana p124 -aVip_src_port -p125 -aVip_dst_addr +sa(dp125 +V_score p126 -aVip_dst_port +F1 +sV_type p127 -aVprotocol +Vvisualization p128 -aVduration +sV_id p129 -aVpkt +VWeb-Request-Type p130 -assV_index +sV_source p131 -V.kibana -p132 -sa(dp133 -V_score +(dp132 +VvisState +p133 +V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} p134 -F1 -sV_type +sVdescription p135 -Vconfig +V +sVtitle p136 -sV_id +VWeb Request Type p137 -V4.5.1 +sVuiStateJSON p138 -sV_source +V{} p139 -(dp140 -VbuildNum +sVversion +p140 +I1 +sVsavedSearchId p141 -I9892 -sVdefaultIndex +Vweb-search p142 -Vbro* +sVkibanaSavedObjectMeta p143 -ssV_index -p144 -V.kibana +(dp144 +VsearchSourceJSON p145 -sa(dp146 -V_score +V{"filter":[]} +p146 +sssV_index p147 -F1 -sV_type +V.kibana p148 -Vindex-pattern -p149 -sV_id +sa(dp149 +V_score p150 -Vbro* +F1 +sV_type p151 -sV_source +Vconfig p152 -(dp153 -Vfields +sV_id +p153 +V4.5.1 p154 -V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +sV_source p155 -sVtimeFieldName -p156 -Vtimestamp +(dp156 +VbuildNum p157 -sVtitle +I9892 +sVdefaultIndex p158 Vbro* p159 @@ -326,25 +326,25 @@ Vvisualization p165 sV_id p166 -VFlow-Duration +VErrors-By-Hostname p167 sV_source p168 (dp169 VvisState p170 -V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}} +V{\u000a "title": "Errors By Error Type",\u000a "type": "histogram",\u000a "params": {\u000a "addLegend": true,\u000a "addTimeMarker": false,\u000a "addTooltip": true,\u000a "defaultYExtents": false,\u000a "mode": "grouped",\u000a "scale": "linear",\u000a "setYExtents": false,\u000a "shareYAxis": true,\u000a "times": [],\u000a "yAxis": {}\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {\u000a "customLabel": "Count"\u000a }\u000a },\u000a {\u000a "id": "2",\u000a "type": "terms",\u000a "schema": "segment",\u000a "params": {\u000a "field": "hostname",\u000a "size": 5,\u000a "order": "desc",\u000a "orderBy": "1"\u000a }\u000a },\u000a {\u000a "id": "4",\u000a "type": "cardinality",\u000a "schema": "metric",\u000a "params": {\u000a "field": "error_hash",\u000a "customLabel": "Unique Datapoint Count"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} p171 sVdescription p172 V sVtitle p173 -VFlow Duration +VErrors By Hostname p174 sVuiStateJSON p175 -V{"vis":{"legendOpen":false}} +V{\u000a "vis": {\u000a "colors": {\u000a "Unique Datapoint Count": "#9AC48A",\u000a "Count": "#629E51"\u000a }\u000a }\u000a} p176 sVversion p177 @@ -354,7 +354,7 @@ p178 (dp179 VsearchSourceJSON p180 -V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "analyze_wildcard": true,\u000a "query": "*"\u000a }\u000a },\u000a "filter": []\u000a} p181 sssV_index p182 @@ -370,25 +370,25 @@ Vvisualization p187 sV_id p188 -VEvents +VWeb-Request-Header p189 sV_source p190 (dp191 VvisState p192 -V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}} p193 sVdescription p194 V sVtitle p195 -VEvents +VWeb Request Header p196 sVuiStateJSON p197 -V{"vis":{"legendOpen":false}} +V{} p198 sVversion p199 @@ -398,7 +398,7 @@ p200 (dp201 VsearchSourceJSON p202 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p203 sssV_index p204 @@ -414,21 +414,21 @@ Vvisualization p209 sV_id p210 -VWeb-Request-Header +VError-Type-Proportion p211 sV_source p212 (dp213 VvisState p214 -V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}} +V{"title":"Error Type Proportion","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"error_type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} p215 sVdescription p216 V sVtitle p217 -VWeb Request Header +VError Type Proportion p218 sVuiStateJSON p219 @@ -442,7 +442,7 @@ p222 (dp223 VsearchSourceJSON p224 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +V{"index":"error*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} p225 sssV_index p226 @@ -458,1082 +458,1884 @@ Vvisualization p231 sV_id p232 -VTop-Alerts-By-Host +VFlow-Duration p233 sV_source p234 (dp235 VvisState p236 -V{"title":"New Visualization","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}} +V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}} p237 sVdescription p238 V sVtitle p239 -VTop Alerts By Host +VFlow Duration p240 sVuiStateJSON p241 -V{} +V{"vis":{"legendOpen":false}} p242 sVversion p243 I1 -sVsavedSearchId -p244 -Vsnort-search -p245 sVkibanaSavedObjectMeta -p246 -(dp247 +p244 +(dp245 VsearchSourceJSON -p248 -V{"filter":[]} -p249 +p246 +V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p247 sssV_index -p250 +p248 V.kibana -p251 -sa(dp252 +p249 +sa(dp250 V_score -p253 +p251 F1 sV_type -p254 +p252 Vvisualization -p255 +p253 sV_id -p256 -VYAF-Flow(s) -p257 +p254 +VErrors-By-Source +p255 sV_source -p258 -(dp259 +p256 +(dp257 VvisState -p260 -V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p261 +p258 +V{"title":"Errors By Source","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source_type","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}}],"listeners":{}} +p259 sVdescription -p262 +p260 V sVtitle -p263 -VYAF Flows -p264 +p261 +VErrors By Source +p262 sVuiStateJSON -p265 +p263 V{} -p266 +p264 sVversion -p267 +p265 I1 sVkibanaSavedObjectMeta -p268 -(dp269 +p266 +(dp267 VsearchSourceJSON -p270 -V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p271 +p268 +V{"index":"error*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p269 sssV_index -p272 +p270 V.kibana -p273 -sa(dp274 +p271 +sa(dp272 V_score -p275 +p273 F1 sV_type -p276 +p274 Vvisualization -p277 +p275 sV_id -p278 -VTop-DNS-Query -p279 +p276 +VEvents +p277 sV_source -p280 -(dp281 +p278 +(dp279 VvisState -p282 -V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} -p283 +p280 +V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +p281 sVdescription -p284 +p282 V sVtitle +p283 +VEvents +p284 +sVuiStateJSON p285 -VTop DNS Query +V{"vis":{"legendOpen":false}} p286 -sVuiStateJSON -p287 -V{} -p288 sVversion -p289 +p287 I1 sVkibanaSavedObjectMeta -p290 -(dp291 +p288 +(dp289 VsearchSourceJSON -p292 -V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p293 +p290 +V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p291 sssV_index -p294 +p292 V.kibana -p295 -sa(dp296 +p293 +sa(dp294 V_score -p297 +p295 F1 sV_type -p298 +p296 Vvisualization -p299 +p297 sV_id -p300 -VTotal-Events -p301 +p298 +VError-Hostname-Proportion +p299 sV_source -p302 -(dp303 +p300 +(dp301 VvisState -p304 -V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}} -p305 +p302 +V{"aggs":[{"id":"1","params":{},"schema":"metric","type":"count"},{"id":"2","params":{"customLabel":"Sensor","field":"hostname","order":"desc","orderBy":"1","size":5},"schema":"segment","type":"terms"}],"listeners":{},"params":{"addLegend":true,"addTooltip":true,"isDonut":false,"shareYAxis":true},"title":"Error Source Proportion","type":"pie"} +p303 sVdescription -p306 +p304 V sVtitle +p305 +VError Hostname Proportion +p306 +sVuiStateJSON p307 -VEvent Count +V{"vis":{"colors":{"host":"#629E51","host2":"#9AC48A","hostAnother":"#7EB26D","hostNew":"#B7DBAB"}}} p308 -sVuiStateJSON -p309 -V{} -p310 sVversion -p311 +p309 I1 sVkibanaSavedObjectMeta -p312 -(dp313 +p310 +(dp311 VsearchSourceJSON -p314 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p315 +p312 +V{"index":"error*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} +p313 sssV_index -p316 +p314 V.kibana -p317 -sa(dp318 +p315 +sa(dp316 V_score -p319 +p317 F1 sV_type -p320 +p318 Vvisualization -p321 +p319 sV_id -p322 -VEvent-Types -p323 +p320 +VUnique-Error-Messages +p321 sV_source -p324 -(dp325 +p322 +(dp323 VvisState -p326 -V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} -p327 +p324 +V{\u000a "title": "Total Unique Error Messages",\u000a "type": "metric",\u000a "params": {\u000a "handleNoResults": true,\u000a "fontSize": 60\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "cardinality",\u000a "schema": "metric",\u000a "params": {\u000a "field": "error_hash",\u000a "customLabel": "Unique Error Messages"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} +p325 sVdescription -p328 +p326 V sVtitle -p329 -VEvent Sources -p330 +p327 +VUnique Error Messages +p328 sVuiStateJSON -p331 +p329 V{} -p332 +p330 sVversion -p333 +p331 I1 sVkibanaSavedObjectMeta -p334 -(dp335 +p332 +(dp333 VsearchSourceJSON -p336 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p337 +p334 +V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "query": "*",\u000a "analyze_wildcard": true\u000a }\u000a },\u000a "filter": []\u000a} +p335 sssV_index -p338 +p336 V.kibana -p339 -sa(dp340 +p337 +sa(dp338 V_score -p341 +p339 F1 sV_type -p342 +p340 Vvisualization -p343 +p341 sV_id -p344 -VUnique-Location(s) -p345 +p342 +VErrors-By-Error-Type +p343 sV_source -p346 -(dp347 +p344 +(dp345 VvisState -p348 -V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}} -p349 +p346 +V{\u000a "title": "Errors By Error Type",\u000a "type": "histogram",\u000a "params": {\u000a "addLegend": true,\u000a "addTimeMarker": false,\u000a "addTooltip": true,\u000a "defaultYExtents": false,\u000a "mode": "grouped",\u000a "scale": "linear",\u000a "setYExtents": false,\u000a "shareYAxis": true,\u000a "times": [],\u000a "yAxis": {}\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {\u000a "customLabel": "Count"\u000a }\u000a },\u000a {\u000a "id": "2",\u000a "type": "terms",\u000a "schema": "segment",\u000a "params": {\u000a "field": "error_type",\u000a "size": 5,\u000a "order": "desc",\u000a "orderBy": "1"\u000a }\u000a },\u000a {\u000a "id": "4",\u000a "type": "cardinality",\u000a "schema": "metric",\u000a "params": {\u000a "field": "error_hash",\u000a "customLabel": "Unique Datapoint Count"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} +p347 sVdescription -p350 +p348 V sVtitle +p349 +VErrors By Error Type +p350 +sVuiStateJSON p351 -VGeo-IP Locations +V{\u000a "vis": {\u000a "colors": {\u000a "Unique Datapoint Count": "#806EB7",\u000a "Count": "#614D93"\u000a }\u000a }\u000a} p352 -sVuiStateJSON -p353 -V{} -p354 sVversion -p355 +p353 I1 sVkibanaSavedObjectMeta -p356 -(dp357 +p354 +(dp355 VsearchSourceJSON -p358 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p359 +p356 +V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "analyze_wildcard": true,\u000a "query": "*"\u000a }\u000a },\u000a "filter": []\u000a} +p357 sssV_index -p360 +p358 V.kibana -p361 -sa(dp362 +p359 +sa(dp360 V_score -p363 +p361 F1 sV_type +p362 +Vsearch +p363 +sV_id p364 -Vvisualization +VErrors p365 -sV_id -p366 -VSnort-Header -p367 sV_source +p366 +(dp367 +Vsort p368 -(dp369 -VvisState +(lp369 +Vtimestamp p370 -V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}} +aVdesc p371 -sVdescription +asVhits p372 +I0 +sVdescription +p373 V sVtitle -p373 -VSnort p374 -sVuiStateJSON +VErrors p375 -V{} -p376 sVversion -p377 +p376 I1 sVkibanaSavedObjectMeta -p378 -(dp379 +p377 +(dp378 VsearchSourceJSON +p379 +V{"index":"error*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} p380 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +ssVcolumns p381 -sssV_index -p382 -V.kibana +(lp382 +Vfailed_sensor_type p383 -sa(dp384 -V_score +aVerror_type +p384 +aVexception p385 +aVhostname +p386 +aVmessage +p387 +aVraw_message +p388 +aVerror_hash +p389 +assV_index +p390 +V.kibana +p391 +sa(dp392 +V_score +p393 F1 sV_type -p386 +p394 Vdashboard -p387 +p395 sV_id -p388 +p396 VMetron-Dashboard -p389 +p397 sV_source -p390 -(dp391 +p398 +(dp399 Vhits -p392 +p400 I0 sVtimeRestore -p393 +p401 I00 sVdescription -p394 +p402 V sVtitle -p395 +p403 VMetron Dashboard -p396 +p404 +sVuiStateJSON +p405 +V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}} +p406 +sVpanelsJSON +p407 +V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","ip_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}] +p408 +sVoptionsJSON +p409 +V{"darkTheme":false} +p410 +sVversion +p411 +I1 +sVkibanaSavedObjectMeta +p412 +(dp413 +VsearchSourceJSON +p414 +V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} +p415 +sssV_index +p416 +V.kibana +p417 +sa(dp418 +V_score +p419 +F1 +sV_type +p420 +Vvisualization +p421 +sV_id +p422 +VSnort-Header +p423 +sV_source +p424 +(dp425 +VvisState +p426 +V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}} +p427 +sVdescription +p428 +V +sVtitle +p429 +VSnort +p430 +sVuiStateJSON +p431 +V{} +p432 +sVversion +p433 +I1 +sVkibanaSavedObjectMeta +p434 +(dp435 +VsearchSourceJSON +p436 +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p437 +sssV_index +p438 +V.kibana +p439 +sa(dp440 +V_score +p441 +F1 +sV_type +p442 +Vvisualization +p443 +sV_id +p444 +VYAF-Flow(s) +p445 +sV_source +p446 +(dp447 +VvisState +p448 +V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p449 +sVdescription +p450 +V +sVtitle +p451 +VYAF Flows +p452 +sVuiStateJSON +p453 +V{} +p454 +sVversion +p455 +I1 +sVkibanaSavedObjectMeta +p456 +(dp457 +VsearchSourceJSON +p458 +V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p459 +sssV_index +p460 +V.kibana +p461 +sa(dp462 +V_score +p463 +F1 +sV_type +p464 +Vvisualization +p465 +sV_id +p466 +VTop-DNS-Query +p467 +sV_source +p468 +(dp469 +VvisState +p470 +V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} +p471 +sVdescription +p472 +V +sVtitle +p473 +VTop DNS Query +p474 +sVuiStateJSON +p475 +V{} +p476 +sVversion +p477 +I1 +sVkibanaSavedObjectMeta +p478 +(dp479 +VsearchSourceJSON +p480 +V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p481 +sssV_index +p482 +V.kibana +p483 +sa(dp484 +V_score +p485 +F1 +sV_type +p486 +Vvisualization +p487 +sV_id +p488 +VEvent-Types +p489 +sV_source +p490 +(dp491 +VvisState +p492 +V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}} +p493 +sVdescription +p494 +V +sVtitle +p495 +VEvent Sources +p496 +sVuiStateJSON +p497 +V{} +p498 +sVversion +p499 +I1 +sVkibanaSavedObjectMeta +p500 +(dp501 +VsearchSourceJSON +p502 +V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p503 +sssV_index +p504 +V.kibana +p505 +sa(dp506 +V_score +p507 +F1 +sV_type +p508 +Vvisualization +p509 +sV_id +p510 +VTotal-Events +p511 +sV_source +p512 +(dp513 +VvisState +p514 +V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}} +p515 +sVdescription +p516 +V +sVtitle +p517 +VEvent Count +p518 +sVuiStateJSON +p519 +V{} +p520 +sVversion +p521 +I1 +sVkibanaSavedObjectMeta +p522 +(dp523 +VsearchSourceJSON +p524 +V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p525 +sssV_index +p526 +V.kibana +p527 +sa(dp528 +V_score +p529 +F1 +sV_type +p530 +Vvisualization +p531 +sV_id +p532 +VUnique-Location(s) +p533 +sV_source +p534 +(dp535 +VvisState +p536 +V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}} +p537 +sVdescription +p538 +V +sVtitle +p539 +VGeo-IP Locations +p540 +sVuiStateJSON +p541 +V{} +p542 +sVversion +p543 +I1 +sVkibanaSavedObjectMeta +p544 +(dp545 +VsearchSourceJSON +p546 +V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p547 +sssV_index +p548 +V.kibana +p549 +sa(dp550 +V_score +p551 +F1 +sV_type +p552 +Vvisualization +p553 +sV_id +p554 +VTop-Alerts-By-Host +p555 +sV_source +p556 +(dp557 +VvisState +p558 +V{"title":"Top Alerts By Host","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}} +p559 +sVdescription +p560 +V +sVtitle +p561 +VTop Alerts By Host +p562 +sVuiStateJSON +p563 +V{} +p564 +sVversion +p565 +I1 +sVsavedSearchId +p566 +Vsnort-search +p567 +sVkibanaSavedObjectMeta +p568 +(dp569 +VsearchSourceJSON +p570 +V{"filter":[]} +p571 +sssV_index +p572 +V.kibana +p573 +sa(dp574 +V_score +p575 +F1 +sV_type +p576 +Vvisualization +p577 +sV_id +p578 +VTotal-Error-Messages +p579 +sV_source +p580 +(dp581 +VvisState +p582 +V{"title":"Total Errored Messages","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Total Error Messages"}}],"listeners":{}} +p583 +sVdescription +p584 +V +sVtitle +p585 +VTotal Error Messages +p586 +sVuiStateJSON +p587 +V{} +p588 +sVversion +p589 +I1 +sVkibanaSavedObjectMeta +p590 +(dp591 +VsearchSourceJSON +p592 +V{"index":"error*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p593 +sssV_index +p594 +V.kibana +p595 +sa(dp596 +V_score +p597 +F1 +sV_type +p598 +Vvisualization +p599 +sV_id +p600 +VErrors-By-Source-Type +p601 +sV_source +p602 +(dp603 +VvisState +p604 +V{\u000a "title": "Errors By Source Type",\u000a "type": "histogram",\u000a "params": {\u000a "shareYAxis": true,\u000a "addTooltip": true,\u000a "addLegend": true,\u000a "scale": "linear",\u000a "mode": "grouped",\u000a "times": [],\u000a "addTimeMarker": false,\u000a "defaultYExtents": false,\u000a "setYExtents": false,\u000a "yAxis": {}\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {\u000a "customLabel": "Count"\u000a }\u000a },\u000a {\u000a "id": "2",\u000a "type": "terms",\u000a "schema": "segment",\u000a "params": {\u000a "field": "failed_sensor_type",\u000a "size": 5,\u000a "order": "desc",\u000a "orderBy": "1"\u000a }\u000a },\u000a {\u000a "id": "4",\u000a "type": "cardinality",\u000a "schema": "metric",\u000a "params": {\u000a "field": "error_hash",\u000a "customLabel": "Unique Datapoint Count"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} +p605 +sVdescription +p606 +V +sVtitle +p607 +VErrors By Source Type +p608 +sVuiStateJSON +p609 +V{\u000a "vis": {\u000a "colors": {\u000a "Unique Datapoint Count": "#0A50A1",\u000a "Count": "#5195CE"\u000a }\u000a }\u000a} +p610 +sVversion +p611 +I1 +sVkibanaSavedObjectMeta +p612 +(dp613 +VsearchSourceJSON +p614 +V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "analyze_wildcard": true,\u000a "query": "*"\u000a }\u000a },\u000a "filter": []\u000a} +p615 +sssV_index +p616 +V.kibana +p617 +sa(dp618 +V_score +p619 +F1 +sV_type +p620 +Vvisualization +p621 +sV_id +p622 +VError-Histogram-By-Sensor-Type +p623 +sV_source +p624 +(dp625 +VvisState +p626 +V{"title":"Error Histogram By Sensor Type","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"grouped","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{},"customLabel":"Time"}},{"id":"3","type":"terms","schema":"group","params":{"field":"failed_sensor_type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +p627 +sVdescription +p628 +V +sVtitle +p629 +VError Histogram By Sensor Type +p630 +sVuiStateJSON +p631 +V{} +p632 +sVversion +p633 +I1 +sVsavedSearchId +p634 +VErrors +p635 +sVkibanaSavedObjectMeta +p636 +(dp637 +VsearchSourceJSON +p638 +V{"filter":[]} +p639 +sssV_index +p640 +V.kibana +p641 +sa(dp642 +V_score +p643 +F1 +sV_type +p644 +Vindex-pattern +p645 +sV_id +p646 +Vsnort* +p647 +sV_source +p648 +(dp649 +Vfields +p650 +V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +p651 +sVtimeFieldName +p652 +Vtimestamp +p653 +sVtitle +p654 +Vsnort* +p655 +ssV_index +p656 +V.kibana +p657 +sa(dp658 +V_score +p659 +F1 +sV_type +p660 +Vindex-pattern +p661 +sV_id +p662 +Vyaf* +p663 +sV_source +p664 +(dp665 +Vfields +p666 +V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +p667 +sVtimeFieldName +p668 +Vtimestamp +p669 +sVtitle +p670 +Vyaf* +p671 +ssV_index +p672 +V.kibana +p673 +sa(dp674 +V_score +p675 +F1 +sV_type +p676 +Vsearch +p677 +sV_id +p678 +Vweb-search +p679 +sV_source +p680 +(dp681 +Vsort +p682 +(lp683 +Vtimestamp +p684 +aVdesc +p685 +asVhits +p686 +I0 +sVdescription +p687 +V +sVtitle +p688 +VWeb Requests +p689 +sVversion +p690 +I1 +sVkibanaSavedObjectMeta +p691 +(dp692 +VsearchSourceJSON +p693 +V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} +p694 +ssVcolumns +p695 +(lp696 +Vmethod +p697 +aVhost +p698 +aVuri +p699 +aVreferrer +p700 +aVip_src_addr +p701 +aVip_dst_addr +p702 +assV_index +p703 +V.kibana +p704 +sa(dp705 +V_score +p706 +F1 +sV_type +p707 +Vvisualization +p708 +sV_id +p709 +VLocation-Header +p710 +sV_source +p711 +(dp712 +VvisState +p713 +V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}} +p714 +sVdescription +p715 +V +sVtitle +p716 +VEnrichment +p717 sVuiStateJSON -p397 -V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}} -p398 -sVpanelsJSON -p399 -V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","ip_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}] -p400 -sVoptionsJSON -p401 -V{"darkTheme":false} -p402 +p718 +V{} +p719 sVversion -p403 +p720 I1 sVkibanaSavedObjectMeta -p404 -(dp405 +p721 +(dp722 VsearchSourceJSON -p406 -V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} -p407 +p723 +V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p724 sssV_index -p408 +p725 V.kibana -p409 -sa(dp410 +p726 +sa(dp727 V_score -p411 +p728 F1 sV_type -p412 +p729 Vvisualization -p413 +p730 sV_id -p414 +p731 VSnort-Alert-Types -p415 +p732 sV_source -p416 -(dp417 +p733 +(dp734 VvisState -p418 +p735 V{"title":"Snort Alert Types","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"sig_id","customLabel":"Alert Type(s)"}}],"listeners":{}} -p419 +p736 sVdescription -p420 +p737 V sVtitle -p421 +p738 VSnort Alert Types -p422 +p739 sVuiStateJSON -p423 +p740 V{} -p424 +p741 sVversion -p425 +p742 I1 sVkibanaSavedObjectMeta -p426 -(dp427 +p743 +(dp744 VsearchSourceJSON -p428 +p745 V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p429 +p746 sssV_index -p430 +p747 V.kibana -p431 -sa(dp432 +p748 +sa(dp749 V_score -p433 +p750 F1 sV_type -p434 +p751 Vvisualization -p435 +p752 sV_id -p436 +p753 VFrequent-DNS-Queries -p437 +p754 sV_source -p438 -(dp439 +p755 +(dp756 VvisState -p440 +p757 V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p441 +p758 sVdescription -p442 +p759 V sVtitle -p443 +p760 VFrequent DNS Requests -p444 +p761 sVuiStateJSON -p445 +p762 V{} -p446 +p763 sVversion -p447 +p764 I1 sVkibanaSavedObjectMeta -p448 -(dp449 +p765 +(dp766 VsearchSourceJSON -p450 +p767 V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p451 +p768 sssV_index -p452 +p769 V.kibana -p453 -sa(dp454 +p770 +sa(dp771 V_score -p455 +p772 F1 sV_type -p456 +p773 Vvisualization -p457 +p774 sV_id -p458 -VLocation-Header -p459 +p775 +VDNS-Request(s) +p776 sV_source -p460 -(dp461 +p777 +(dp778 VvisState -p462 -V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}} -p463 +p779 +V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p780 sVdescription -p464 +p781 V sVtitle -p465 -VEnrichment -p466 +p782 +VDNS Requests +p783 sVuiStateJSON -p467 +p784 V{} -p468 +p785 sVversion -p469 +p786 I1 +sVsavedSearchId +p787 +Vdns-search +p788 sVkibanaSavedObjectMeta -p470 -(dp471 +p789 +(dp790 VsearchSourceJSON -p472 -V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p473 +p791 +V{"filter":[]} +p792 sssV_index -p474 +p793 V.kibana -p475 -sa(dp476 +p794 +sa(dp795 V_score -p477 +p796 F1 sV_type -p478 -Vsearch -p479 +p797 +Vvisualization +p798 sV_id -p480 -Vweb-search -p481 +p799 +VHTTP(S)-Requests +p800 sV_source -p482 -(dp483 -Vsort -p484 -(lp485 -Vtimestamp -p486 -aVdesc -p487 -asVhits -p488 -I0 +p801 +(dp802 +VvisState +p803 +V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} +p804 sVdescription -p489 +p805 V sVtitle -p490 +p806 VWeb Requests -p491 +p807 +sVuiStateJSON +p808 +V{} +p809 sVversion -p492 +p810 I1 +sVsavedSearchId +p811 +Vweb-search +p812 sVkibanaSavedObjectMeta -p493 -(dp494 +p813 +(dp814 VsearchSourceJSON -p495 -V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p496 -ssVcolumns -p497 -(lp498 -Vmethod -p499 -aVhost -p500 -aVuri -p501 -aVreferrer -p502 -aVip_src_addr -p503 -aVip_dst_addr -p504 -assV_index -p505 +p815 +V{"filter":[]} +p816 +sssV_index +p817 V.kibana -p506 -sa(dp507 +p818 +sa(dp819 V_score -p508 +p820 F1 sV_type -p509 -Vindex-pattern -p510 +p821 +Vvisualization +p822 sV_id -p511 -Vsnort* -p512 +p823 +VErrors-Over-Time +p824 sV_source -p513 -(dp514 -Vfields -p515 -V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p516 -sVtimeFieldName -p517 -Vtimestamp -p518 +p825 +(dp826 +VvisState +p827 +V{\u000a "title": "Error Over Time",\u000a "type": "line",\u000a "params": {\u000a "shareYAxis": true,\u000a "addTooltip": true,\u000a "addLegend": true,\u000a "showCircles": true,\u000a "smoothLines": false,\u000a "interpolate": "linear",\u000a "scale": "linear",\u000a "drawLinesBetweenPoints": true,\u000a "radiusRatio": 9,\u000a "times": [],\u000a "addTimeMarker": true,\u000a "defaultYExtents": false,\u000a "setYExtents": false,\u000a "yAxis": {\u000a "min": 0\u000a }\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {}\u000a },\u000a {\u000a "id": "2",\u000a "type": "date_histogram",\u000a "schema": "segment",\u000a "params": {\u000a "field": "timestamp",\u000a "interval": "auto",\u000a "customInterval": "2h",\u000a "min_doc_count": 1,\u000a "extended_bounds": {}\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} +p828 +sVdescription +p829 +V sVtitle -p519 -Vsnort* -p520 -ssV_index -p521 +p830 +VErrors Over Time +p831 +sVuiStateJSON +p832 +V{} +p833 +sVversion +p834 +I1 +sVkibanaSavedObjectMeta +p835 +(dp836 +VsearchSourceJSON +p837 +V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "query": "*",\u000a "analyze_wildcard": true\u000a }\u000a },\u000a "filter": []\u000a} +p838 +sssV_index +p839 V.kibana -p522 -sa(dp523 +p840 +sa(dp841 V_score -p524 +p842 F1 sV_type -p525 +p843 +Vvisualization +p844 +sV_id +p845 +VError-Source-Proportion +p846 +sV_source +p847 +(dp848 +VvisState +p849 +V{\u000a "title": "Sensor Type Proportion",\u000a "type": "pie",\u000a "params": {\u000a "shareYAxis": true,\u000a "addTooltip": true,\u000a "addLegend": true,\u000a "isDonut": false\u000a },\u000a "aggs": [\u000a {\u000a "id": "1",\u000a "type": "count",\u000a "schema": "metric",\u000a "params": {}\u000a },\u000a {\u000a "id": "2",\u000a "type": "terms",\u000a "schema": "segment",\u000a "params": {\u000a "field": "failed_sensor_type",\u000a "size": 5,\u000a "order": "desc",\u000a "orderBy": "1",\u000a "customLabel": "Sensor"\u000a }\u000a }\u000a ],\u000a "listeners": {}\u000a} +p850 +sVdescription +p851 +V +sVtitle +p852 +VError Source Proportion +p853 +sVuiStateJSON +p854 +V{} +p855 +sVversion +p856 +I1 +sVkibanaSavedObjectMeta +p857 +(dp858 +VsearchSourceJSON +p859 +V{\u000a "index": "error*",\u000a "query": {\u000a "query_string": {\u000a "query": "*",\u000a "analyze_wildcard": true\u000a }\u000a },\u000a "filter": []\u000a} +p860 +sssV_index +p861 +V.kibana +p862 +sa(dp863 +V_score +p864 +F1 +sV_type +p865 Vindex-pattern -p526 +p866 sV_id -p527 -Vyaf* -p528 +p867 +Verror* +p868 sV_source -p529 -(dp530 +p869 +(dp870 Vfields -p531 -V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] -p532 +p871 +V[{"name":"exception","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"stack","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"error_hash","type":"string","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"raw_message","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"message","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"failed_sensor_type","type":"string","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"hostname","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":1,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"error_type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"error_fields","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"raw_message_bytes","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"timestamp","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}] +p872 sVtimeFieldName -p533 +p873 Vtimestamp -p534 +p874 sVtitle -p535 -Vyaf* -p536 +p875 +Verror* +p876 ssV_index -p537 +p877 V.kibana -p538 -sa(dp539 +p878 +sa(dp879 V_score -p540 +p880 F1 sV_type -p541 +p881 Vvisualization -p542 +p882 sV_id -p543 -VDNS-Request(s) -p544 +p883 +VError-Date-Histogram +p884 sV_source -p545 -(dp546 +p885 +(dp886 VvisState -p547 -V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p548 +p887 +V{"title":"New Visualization","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{},"customLabel":"Time"}}],"listeners":{}} +p888 sVdescription -p549 +p889 V sVtitle -p550 -VDNS Requests -p551 +p890 +VError Date Histogram +p891 sVuiStateJSON -p552 +p892 V{} -p553 +p893 sVversion -p554 +p894 I1 sVsavedSearchId -p555 -Vdns-search -p556 +p895 +VErrors +p896 sVkibanaSavedObjectMeta -p557 -(dp558 +p897 +(dp898 VsearchSourceJSON -p559 +p899 V{"filter":[]} -p560 +p900 sssV_index -p561 +p901 V.kibana -p562 -sa(dp563 +p902 +sa(dp903 V_score -p564 +p904 F1 sV_type -p565 -Vvisualization -p566 +p905 +Vdashboard +p906 sV_id -p567 -VHTTP(S)-Requests -p568 +p907 +VMetron-Error-Dashboard +p908 sV_source -p569 -(dp570 -VvisState -p571 -V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}} -p572 +p909 +(dp910 +Vhits +p911 +I0 +sVtimeRestore +p912 +I00 sVdescription -p573 +p913 V sVtitle -p574 -VWeb Requests -p575 +p914 +VMetron Error Dashboard +p915 sVuiStateJSON -p576 -V{} -p577 +p916 +V{"P-2":{"vis":{"legendOpen":true}},"P-23":{"vis":{"colors":{"amb3.service.consul":"#629E51","host":"#629E51","host2":"#9AC48A","hostAnother":"#7EB26D","hostNew":"#B7DBAB"}}},"P-3":{"vis":{"colors":{"fourth":"#1F78C1","new_error":"#BADFF4","test_error":"#82B5D8"}}},"P-5":{"vis":{"colors":{"another_new_parser_error":"#806EB7","new_parser_error":"#AEA2E0","parser_error":"#614D93"}}}} +p917 +sVpanelsJSON +p918 +V[{"col":5,"id":"Errors-By-Error-Type","panelIndex":2,"row":9,"size_x":8,"size_y":3,"type":"visualization"},{"col":1,"id":"Error-Source-Proportion","panelIndex":3,"row":9,"size_x":4,"size_y":3,"type":"visualization"},{"col":5,"id":"Errors-By-Source-Type","panelIndex":4,"row":12,"size_x":8,"size_y":3,"type":"visualization"},{"col":1,"id":"Error-Type-Proportion","panelIndex":5,"row":12,"size_x":4,"size_y":3,"type":"visualization"},{"col":8,"id":"Unique-Error-Messages","panelIndex":19,"row":1,"size_x":4,"size_y":2,"type":"visualization"},{"col":3,"id":"Total-Error-Messages","panelIndex":20,"row":1,"size_x":4,"size_y":2,"type":"visualization"},{"col":5,"id":"Errors-By-Hostname","panelIndex":22,"row":15,"size_x":8,"size_y":3,"type":"visualization"},{"col":1,"id":"Error-Hostname-Proportion","panelIndex":23,"row":15,"size_x":4,"size_y":3,"type":"visualization"},{"col":1,"columns":["failed_sensor_type","error_type","exception","hostname","message","raw_message","error_hash"],"id":"Errors","panelIndex":25,"row":18,"size_x":12,"size_y":7,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Error-Histogram-By-Sensor-Type","panelIndex":27,"row":3,"size_x":12,"size_y":3,"type":"visualization"},{"id":"Unique-Error-Histogram-By-Sensor-Type","type":"visualization","panelIndex":28,"size_x":12,"size_y":3,"col":1,"row":6}] +p919 +sVoptionsJSON +p920 +V{"darkTheme":false} +p921 sVversion -p578 +p922 I1 -sVsavedSearchId -p579 -Vweb-search -p580 sVkibanaSavedObjectMeta -p581 -(dp582 +p923 +(dp924 VsearchSourceJSON -p583 -V{"filter":[]} -p584 +p925 +V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]} +p926 sssV_index -p585 +p927 V.kibana -p586 -sa(dp587 +p928 +sa(dp929 V_score -p588 +p930 F1 sV_type -p589 +p931 +Vconfig +p932 +sV_id +p933 +V4.5.3 +p934 +sV_source +p935 +(dp936 +VbuildNum +p937 +I9892 +sVdefaultIndex +p938 +Vbro* +p939 +ssV_index +p940 +V.kibana +p941 +sa(dp942 +V_score +p943 +F1 +sV_type +p944 Vsearch -p590 +p945 sV_id -p591 +p946 Vdns-search -p592 +p947 sV_source -p593 -(dp594 +p948 +(dp949 Vsort -p595 -(lp596 +p950 +(lp951 Vtimestamp -p597 +p952 aVdesc -p598 +p953 asVhits -p599 +p954 I0 sVdescription -p600 +p955 V sVtitle -p601 +p956 VDNS Requests -p602 +p957 sVversion -p603 +p958 I1 sVkibanaSavedObjectMeta -p604 -(dp605 +p959 +(dp960 VsearchSourceJSON -p606 +p961 V{"index":"bro*","query":{"query_string":{"query":"protocol: dns","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}} -p607 +p962 ssVcolumns -p608 -(lp609 +p963 +(lp964 Vquery -p610 +p965 aVqtype_name -p611 +p966 aVanswers -p612 +p967 aVip_src_addr -p613 +p968 aVip_dst_addr -p614 +p969 assV_index -p615 +p970 V.kibana -p616 -sa(dp617 +p971 +sa(dp972 V_score -p618 +p973 F1 sV_type -p619 +p974 Vvisualization -p620 +p975 sV_id -p621 -VFlow-Locations -p622 +p976 +VDNS-Requests-Header +p977 sV_source -p623 -(dp624 +p978 +(dp979 VvisState -p625 -V{"title":"New Visualization","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":false,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}} -p626 +p980 +V{"aggs":[],"listeners":{},"params":{"markdown":"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network."},"title":"DNS Requests","type":"markdown"} +p981 sVdescription -p627 +p982 V sVtitle -p628 -VFlow Locations -p629 +p983 +VDNS Requests +p984 sVuiStateJSON -p630 +p985 V{} -p631 +p986 sVversion -p632 +p987 I1 sVkibanaSavedObjectMeta -p633 -(dp634 +p988 +(dp989 VsearchSourceJSON -p635 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p636 +p990 +V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} +p991 sssV_index -p637 +p992 V.kibana -p638 -sa(dp639 +p993 +sa(dp994 V_score -p640 +p995 F1 sV_type -p641 +p996 Vvisualization -p642 +p997 sV_id -p643 -VUnusual-Referrers -p644 +p998 +VYAF-Flows-Header +p999 sV_source -p645 -(dp646 +p1000 +(dp1001 VvisState -p647 -V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}} -p648 +p1002 +V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}} +p1003 sVdescription -p649 +p1004 V sVtitle -p650 -VUnusual Referrers -p651 +p1005 +VYAF +p1006 sVuiStateJSON -p652 +p1007 V{} -p653 +p1008 sVversion -p654 +p1009 I1 -sVsavedSearchId -p655 -Vweb-search -p656 sVkibanaSavedObjectMeta -p657 -(dp658 +p1010 +(dp1011 VsearchSourceJSON -p659 -V{"filter":[]} -p660 +p1012 +V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} +p1013 sssV_index -p661 +p1014 V.kibana -p662 -sa(dp663 +p1015 +sa(dp1016 V_score -p664 +p1017 F1 sV_type -p665 +p1018 Vvisualization -p666 +p1019 sV_id -p667 +p1020 +VTop-5-Exceptions +p1021 +sV_source +p1022 +(dp1023 +VvisState +p1024 +V{"title":"Top-5 Exceptions","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"exception","size":5,"order":"desc","orderBy":"1","customLabel":"Exceptions"}}],"listeners":{}} +p1025 +sVdescription +p1026 +V +sVtitle +p1027 +VTop-5 Exceptions +p1028 +sVuiStateJSON +p1029 +V{} +p1030 +sVversion +p1031 +I1 +sVkibanaSavedObjectMeta +p1032 +(dp1033 +VsearchSourceJSON +p1034 +V{"index":"error*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p1035 +sssV_index +p1036 +V.kibana +p1037 +sa(dp1038 +V_score +p1039 +F1 +sV_type +p1040 +Vvisualization +p1041 +sV_id +p1042 VFrequent-DNS-Requests -p668 +p1043 sV_source -p669 -(dp670 +p1044 +(dp1045 VvisState -p671 +p1046 V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1","customLabel":"DNS Query"}}],"listeners":{}} -p672 +p1047 sVdescription -p673 +p1048 V sVtitle -p674 +p1049 VFrequent DNS Requests -p675 +p1050 sVuiStateJSON -p676 +p1051 V{} -p677 +p1052 sVversion -p678 +p1053 I1 sVkibanaSavedObjectMeta -p679 -(dp680 +p1054 +(dp1055 VsearchSourceJSON -p681 +p1056 V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p682 +p1057 sssV_index -p683 +p1058 V.kibana -p684 -sa(dp685 +p1059 +sa(dp1060 V_score -p686 +p1061 F1 sV_type -p687 +p1062 Vvisualization -p688 +p1063 sV_id -p689 +p1064 VCountry -p690 +p1065 sV_source -p691 -(dp692 +p1066 +(dp1067 VvisState -p693 +p1068 V{"title":"By Country","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"enrichments:geo:ip_src_addr:country","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} -p694 +p1069 sVdescription -p695 +p1070 V sVtitle -p696 +p1071 VBy Country -p697 +p1072 sVuiStateJSON -p698 +p1073 V{} -p699 +p1074 sVversion -p700 +p1075 I1 sVkibanaSavedObjectMeta -p701 -(dp702 +p1076 +(dp1077 VsearchSourceJSON -p703 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p704 +p1078 +V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p1079 sssV_index -p705 +p1080 V.kibana -p706 -sa(dp707 +p1081 +sa(dp1082 V_score -p708 +p1083 F1 sV_type -p709 +p1084 Vvisualization -p710 +p1085 sV_id -p711 +p1086 +VFlow-Locations +p1087 +sV_source +p1088 +(dp1089 +VvisState +p1090 +V{"title":"Flow Locations","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":false,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}} +p1091 +sVdescription +p1092 +V +sVtitle +p1093 +VFlow Locations +p1094 +sVuiStateJSON +p1095 +V{} +p1096 +sVversion +p1097 +I1 +sVkibanaSavedObjectMeta +p1098 +(dp1099 +VsearchSourceJSON +p1100 +V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p1101 +sssV_index +p1102 +V.kibana +p1103 +sa(dp1104 +V_score +p1105 +F1 +sV_type +p1106 +Vvisualization +p1107 +sV_id +p1108 VTop-Destinations -p712 +p1109 sV_source -p713 -(dp714 +p1110 +(dp1111 VvisState -p715 +p1112 V{"title":"Top Destinations","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Destination IP"}}],"listeners":{}} -p716 +p1113 sVdescription -p717 +p1114 V sVtitle -p718 +p1115 VTop Destinations -p719 +p1116 sVuiStateJSON -p720 +p1117 V{} -p721 +p1118 sVversion -p722 +p1119 I1 sVkibanaSavedObjectMeta -p723 -(dp724 +p1120 +(dp1121 VsearchSourceJSON -p725 -V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} -p726 +p1122 +V{"index":["yaf*","bro*","snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]} +p1123 sssV_index -p727 +p1124 V.kibana -p728 -sa(dp729 +p1125 +sa(dp1126 V_score -p730 +p1127 F1 sV_type -p731 +p1128 Vvisualization -p732 +p1129 sV_id -p733 -VDNS-Requests-Header -p734 +p1130 +VUnusual-Referrers +p1131 sV_source -p735 -(dp736 +p1132 +(dp1133 VvisState -p737 -V{"aggs":[],"listeners":{},"params":{"markdown":"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network."},"title":"DNS Requests","type":"markdown"} -p738 +p1134 +V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}} +p1135 sVdescription -p739 +p1136 V sVtitle -p740 -VDNS Requests -p741 +p1137 +VUnusual Referrers +p1138 sVuiStateJSON -p742 +p1139 V{} -p743 +p1140 sVversion -p744 +p1141 I1 +sVsavedSearchId +p1142 +Vweb-search +p1143 sVkibanaSavedObjectMeta -p745 -(dp746 +p1144 +(dp1145 VsearchSourceJSON -p747 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p748 +p1146 +V{"filter":[]} +p1147 sssV_index -p749 +p1148 V.kibana -p750 -sa(dp751 +p1149 +sa(dp1150 V_score -p752 +p1151 F1 sV_type -p753 +p1152 Vvisualization -p754 +p1153 sV_id -p755 -VYAF-Flows-Header -p756 +p1154 +VUnique-Error-Histogram-By-Sensor-Type +p1155 sV_source -p757 -(dp758 +p1156 +(dp1157 VvisState -p759 -V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}} -p760 +p1158 +V{"title":"Error Histogram By Sensor Type","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"grouped","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"error_hash"}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{},"customLabel":"Time"}},{"id":"3","type":"terms","schema":"group","params":{"field":"failed_sensor_type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}} +p1159 sVdescription -p761 +p1160 V sVtitle -p762 -VYAF -p763 +p1161 +VUnique Error Histogram By Sensor Type +p1162 sVuiStateJSON -p764 +p1163 V{} -p765 +p1164 sVversion -p766 +p1165 I1 +sVsavedSearchId +p1166 +VErrors +p1167 sVkibanaSavedObjectMeta -p767 -(dp768 +p1168 +(dp1169 VsearchSourceJSON -p769 -V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]} -p770 +p1170 +V{"filter":[]} +p1171 sssV_index -p771 +p1172 V.kibana -p772 +p1173 sa. \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index eeb20372db..cbff4a90e4 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -150,7 +150,8 @@ { "es.clustername": "{{ es_cluster_name }}", "es.ip": "{{ es_url }}", -"es.date.format": "yyyy.MM.dd.HH" +"es.date.format": "yyyy.MM.dd.HH", +"parser.error.topic": "indexing" } @@ -171,7 +172,7 @@ kafka.broker={{ kafka_brokers }} kafka.start=WHERE_I_LEFT_OFF ##### Indexing ##### index.input.topic=indexing -index.error.topic=indexing_error +index.error.topic=indexing writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter ##### Metrics ##### #reporters diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template new file mode 100644 index 0000000000..3bb4633704 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template @@ -0,0 +1,57 @@ +{ + "template": "error_index*", + "mappings": { + "error_doc": { + "_timestamp": { + "enabled": true + }, + "properties": { + "exception": { + "type": "string", + "index": "not_analyzed" + }, + "hostname": { + "type": "string", + "index": "not_analyzed" + }, + "stack": { + "type": "string", + "index": "not_analyzed" + }, + "timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "message": { + "type": "string", + "index": "not_analyzed" + }, + "raw_message": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 8191 + }, + "raw_message_bytes": { + "type": "binary", + "index": "no" + }, + "error_fields": { + "type": "string", + "index": "not_analyzed" + }, + "error_hash": { + "type": "string", + "index": "not_analyzed" + }, + "failed_sensor_type": { + "type": "string", + "index": "not_analyzed" + }, + "error_type": { + "type": "string", + "index": "not_analyzed" + } + } + } + } +} diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index bc73c87eb3..817f2668f2 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -28,8 +28,6 @@ class EnrichmentCommands: __params = None __enrichment_topology = None __enrichment_topic = None - __enrichment_error_topic = None - __threat_intel_error_topic = None __configured = False def __init__(self, params): @@ -38,8 +36,6 @@ def __init__(self, params): self.__params = params self.__enrichment_topology = params.metron_enrichment_topology self.__enrichment_topic = params.metron_enrichment_topic - self.__enrichment_error_topic = params.metron_enrichment_error_topic - self.__threat_intel_error_topic = params.metron_threat_intel_error_topic self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file) def is_configured(self): @@ -121,7 +117,7 @@ def init_kafka_topics(self): retention_bytes = retention_gigabytes * 1024 * 1024 * 1024 Logger.info("Creating topics for enrichment") - topics = [self.__enrichment_topic, self.__enrichment_error_topic, self.__threat_intel_error_topic] + topics = [self.__enrichment_topic] for topic in topics: Logger.info("Creating topic'{0}'".format(topic)) Execute(command_template.format(self.__params.kafka_bin_dir, diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index efc048d93a..53fb17ba99 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -95,6 +95,11 @@ def elasticsearch_template_install(self, env): content=StaticFile('yaf_index.template') ) + File(params.error_index_path, + mode=0755, + content=StaticFile('error_index.template') + ) + bro_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}') Execute(bro_cmd, logoutput=True) @@ -104,6 +109,9 @@ def elasticsearch_template_install(self, env): yaf_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}') Execute(yaf_cmd, logoutput=True) + error_cmd = ambari_format( + 'curl -s -XPOST http://{es_http_url}/_template/error_index -d @{error_index_path}') + Execute(error_cmd, logoutput=True) def elasticsearch_template_delete(self, env): from params import params @@ -115,6 +123,8 @@ def elasticsearch_template_delete(self, env): Execute(snort_cmd, logoutput=True) yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"') Execute(yaf_cmd, logoutput=True) + error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/error_index*"') + Execute(error_cmd, logoutput=True) def zeppelin_notebook_import(self, env): from params import params diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 2427d255f9..2b8276b59f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -158,13 +158,12 @@ metron_enrichment_topology = status_params.metron_enrichment_topology metron_enrichment_topic = status_params.metron_enrichment_topic -metron_enrichment_error_topic = status_params.metron_enrichment_error_topic -metron_threat_intel_error_topic = status_params.metron_threat_intel_error_topic # ES Templates bro_index_path = tmp_dir + "/bro_index.template" snort_index_path = tmp_dir + "/snort_index.template" yaf_index_path = tmp_dir + "/yaf_index.template" +error_index_path = tmp_dir + "/error_index.template" # Zeppelin Notebooks metron_config_zeppelin_path = format("{metron_config_path}/zeppelin") diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index e8a8568e4c..961102fdcb 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -34,8 +34,6 @@ # Enrichment metron_enrichment_topology = 'enrichment' metron_enrichment_topic = 'enrichments' -metron_enrichment_error_topic = 'enrichments_error' -metron_threat_intel_error_topic = 'threatintel_error' enrichment_table = 'enrichment' enrichment_cf = 't' diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py index e6f0f3a704..574d7e81a9 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py @@ -118,18 +118,6 @@ def init_kafka_topics(self): num_partitions, replication_factor, retention_bytes)) - Logger.info("Creating topics for error handling") - Execute(command_template.format(self.__params.kafka_bin_dir, - self.__params.zookeeper_quorum, - "parser_invalid", - num_partitions, - replication_factor, - retention_bytes)) - Execute(command_template.format(self.__params.kafka_bin_dir, - self.__params.zookeeper_quorum, - "parser_error", - num_partitions, replication_factor, - retention_bytes)) Logger.info("Done creating Kafka topics") def start_parser_topologies(self): diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 3e8a11cd02..2c619e11a9 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -260,6 +260,7 @@ This package installs the Metron Indexing files %{metron_home}/config/zookeeper/indexing/websphere.json %{metron_home}/config/zookeeper/indexing/yaf.json %{metron_home}/config/zookeeper/indexing/asa.json +%{metron_home}/config/zookeeper/indexing/error.json %{metron_home}/config/zeppelin/metron/metron-yaf-telemetry.json # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/error_index.template b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/error_index.template new file mode 100644 index 0000000000..070c90f369 --- /dev/null +++ b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/error_index.template @@ -0,0 +1,53 @@ +{ + "template": "error_index*", + "mappings": { + "error_doc": { + "_timestamp": { + "enabled": true + }, + "properties": { + "exception": { + "type": "string", + "index": "not_analyzed" + }, + "hostname": { + "type": "string", + "index": "not_analyzed" + }, + "stack": { + "type": "string", + "index": "not_analyzed" + }, + "time": { + "type": "date", + "format": "epoch_millis" + }, + "message": { + "type": "string", + "index": "not_analyzed" + }, + "raw_message": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 8191 + }, + "raw_message_bytes": { + "type": "binary", + "index": "no" + }, + "raw_message_hash": { + "type": "string", + "index": "not_analyzed" + }, + "source_type": { + "type": "string", + "index": "not_analyzed" + }, + "error_type": { + "type": "string", + "index": "not_analyzed" + } + } + } + } +} diff --git a/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json b/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json index 817710247f..87af1c0de8 100644 --- a/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json +++ b/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json @@ -2,5 +2,6 @@ "es.clustername": "{{ elasticsearch_cluster_name }}", "es.ip": "{{ groups.search[0] }}", "es.port": "{{ elasticsearch_transport_port }}", - "es.date.format": "yyyy.MM.dd.HH" + "es.date.format": "yyyy.MM.dd.HH", + "parser.error.topic": "indexing" } diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md index 97ecd58b27..d66ab0caea 100644 --- a/metron-platform/metron-common/README.md +++ b/metron-platform/metron-common/README.md @@ -784,6 +784,7 @@ This configuration is stored in zookeeper, but looks something like "es.ip": "node1", "es.port": "9300", "es.date.format": "yyyy.MM.dd.HH", + "parser.error.topic": "indexing" "fieldValidations" : [ { "input" : [ "ip_src_addr", "ip_dst_addr" ], @@ -901,3 +902,39 @@ Usage examples: * To dump the existing configs from zookeeper on the singlenode vagrant machine: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m DUMP` * To push the configs into zookeeper on the singlenode vagrant machine: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper` * To pull the configs from zookeeper to the singlenode vagrant machine disk: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PULL -o $METRON_HOME/config/zookeeper -f` + +# Topology Errors + +Errors generated in Metron topologies are transformed into JSON format and follow this structure: + +``` +{ + "exception": "java.lang.IllegalStateException: Unable to parse Message: ...", + "failed_sensor_type": "bro", + "stack": "java.lang.IllegalStateException: Unable to parse Message: ...", + "hostname": "node1", + "source:type": "error", + "raw_message": "{\"http\": {\"ts\":1488809627.000000.31915,\"uid\":\"C9JpSd2vFAWo3mXKz1\", ...", + "error_hash": "f7baf053f2d3c801a01d196f40f3468e87eea81788b2567423030100865c5061", + "error_type": "parser_error", + "message": "Unable to parse Message: {\"http\": {\"ts\":1488809627.000000.31915,\"uid\":\"C9JpSd2vFAWo3mXKz1\", ...", + "timestamp": 1488809630698 +} +``` + +Each topology can be configured to send error messages to a specific Kafka topic. The parser topologies retrieve this setting from the the `parser.error.topic` setting in the global config: +``` +{ + "es.clustername": "metron", + "es.ip": "node1", + "es.port": "9300", + "es.date.format": "yyyy.MM.dd.HH", + "parser.error.topic": "indexing" +} +``` + +Error topics for enrichment and threat intel errors are passed into the enrichment topology as flux properties named `enrichment.error.topic` and `threat.intel.error.topic`. These properties can be found in `$METRON_HOME/config/enrichment.properties`. + +The error topic for indexing errors is passed into the indexing topology as a flux property named `index.error.topic`. This property can be found in either `$METRON_HOME/config/elasticsearch.properties` or `$METRON_HOME/config/solr.properties` depending on the search engine selected. + +By default all error messages are sent to the `indexing` topic so that they are indexed and archived, just like other messages. The indexing config for error messages can be found at `$METRON_HOME/config/zookeeper/indexing/error.json`. diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index d51762abf6..8d1e183d24 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -308,7 +308,6 @@ - org.apache.curator curator-test @@ -363,7 +362,12 @@ stream 2.9.5 - + + org.apache.metron + metron-test-utilities + ${project.parent.version} + test + diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java index 4230678f36..29be31e8e3 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java @@ -27,14 +27,8 @@ public class Constants { public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; public static final String SENSOR_TYPE = "source.type"; public static final String ENRICHMENT_TOPIC = "enrichments"; - public static final String ENRICHMENT_ERROR_TOPIC = "enrichments_error"; - public static final String THREAT_INTEL_ERROR_TOPIC = "threatintel_error"; public static final String INDEXING_TOPIC = "indexing"; - public static final String INDEXING_ERROR_TOPIC = "indexing_error"; - public static final String DEFAULT_PARSER_ERROR_TOPIC = "parser_error"; - public static final String DEFAULT_PARSER_INVALID_TOPIC = "parser_invalid"; public static final String ERROR_STREAM = "error"; - public static final String INVALID_STREAM = "invalid"; public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment"; public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel"; @@ -72,5 +66,51 @@ public static Fields fromString(String fieldName) { } } + public enum ErrorFields { + MESSAGE("message") + ,FAILED_SENSOR_TYPE("failed_sensor_type") + ,ERROR_TYPE("error_type") + ,EXCEPTION("exception") + ,STACK("stack") + ,TIMESTAMP("timestamp") + ,HOSTNAME("hostname") + ,RAW_MESSAGE("raw_message") + ,RAW_MESSAGE_BYTES("raw_message_bytes") + ,ERROR_FIELDS("error_fields") + ,ERROR_HASH("error_hash") + ; + + private String name; + + ErrorFields(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + public enum ErrorType { + + PARSER_ERROR("parser_error") + ,PARSER_INVALID("parser_invalid") + ,ENRICHMENT_ERROR("enrichments_error") + ,THREAT_INTEL_ERROR("threatintel_error") + ,INDEXING_ERROR("indexing_error") + ,DEFAULT_ERROR("error") + ; + + private String type; + + ErrorType(String type) { + this.type = type; + } + + public String getType() { + return type; + } + } + } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java index 3302426ad0..970566c4fa 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java @@ -66,6 +66,8 @@ else if(inputObj instanceof List) { for(Object inputO : (List)inputObj) { input.add(inputO.toString()); } + } else { + input = new ArrayList<>(); } config = Config.CONFIG.get(validatorConfig, Map.class); if(config == null) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java new file mode 100644 index 0000000000..2837d34382 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.error; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.metron.common.Constants; +import org.apache.metron.common.Constants.ErrorType; +import org.apache.metron.common.utils.HashUtils; +import org.json.simple.JSONObject; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.metron.common.Constants.ErrorFields; + +public class MetronError { + + private String message; + private Throwable throwable; + private String sensorType = "error"; + private ErrorType errorType = ErrorType.DEFAULT_ERROR; + private Set errorFields; + private List rawMessages; + + public MetronError withMessage(String message) { + this.message = message; + return this; + } + + public MetronError withThrowable(Throwable throwable) { + this.throwable = throwable; + return this; + } + + public MetronError withSensorType(String sensorType) { + this.sensorType = sensorType; + return this; + } + + public MetronError withErrorType(ErrorType errorType) { + this.errorType = errorType; + return this; + } + + public MetronError withErrorFields(Set errorFields) { + this.errorFields = errorFields; + return this; + } + + + public MetronError addRawMessage(Object rawMessage) { + if (rawMessage != null) { + if (this.rawMessages == null) { + this.rawMessages = new ArrayList<>(); + } + this.rawMessages.add(rawMessage); + } + return this; + } + + public MetronError withRawMessages(List rawMessages) { + this.rawMessages = rawMessages; + return this; + } + + public Optional getThrowable() { + return throwable != null ? Optional.of(throwable) : Optional.empty(); + } + + @SuppressWarnings({"unchecked"}) + public JSONObject getJSONObject() { + JSONObject errorMessage = new JSONObject(); + errorMessage.put(Constants.SENSOR_TYPE, "error"); + errorMessage.put(ErrorFields.FAILED_SENSOR_TYPE.getName(), sensorType); + errorMessage.put(ErrorFields.ERROR_TYPE.getName(), errorType.getType()); + + addMessageString(errorMessage); + addStacktrace(errorMessage); + addTimestamp(errorMessage); + addHostname(errorMessage); + addRawMessages(errorMessage); + addErrorHash(errorMessage); + + return errorMessage; + } + + @SuppressWarnings({"unchecked"}) + private void addMessageString(JSONObject errorMessage) { + if (message != null) { + errorMessage.put(ErrorFields.MESSAGE.getName(), message); + } else if (throwable != null) { + errorMessage.put(ErrorFields.MESSAGE.getName(), throwable.getMessage()); + } + } + + @SuppressWarnings({"unchecked"}) + private void addStacktrace(JSONObject errorMessage) { + if (throwable != null) { + String stackTrace = ExceptionUtils.getStackTrace(throwable); + String exception = throwable.toString(); + errorMessage.put(ErrorFields.EXCEPTION.getName(), exception); + errorMessage.put(ErrorFields.STACK.getName(), stackTrace); + } + } + + @SuppressWarnings({"unchecked"}) + private void addTimestamp(JSONObject errorMessage) { + errorMessage.put(ErrorFields.TIMESTAMP.getName(), System.currentTimeMillis()); + } + + @SuppressWarnings({"unchecked"}) + private void addHostname(JSONObject errorMessage) { + try { + errorMessage.put(ErrorFields.HOSTNAME.getName(), InetAddress.getLocalHost().getHostName()); + } catch (UnknownHostException ex) { + // Leave the hostname field off if it cannot be found + } + } + + @SuppressWarnings({"unchecked"}) + private void addRawMessages(JSONObject errorMessage) { + if(rawMessages != null) { + for(int i = 0; i < rawMessages.size(); i++) { + Object rawMessage = rawMessages.get(i); + // If multiple messages are included add an index to the field name, otherwise leave it off + String rawMessageField = rawMessages.size() == 1 ? ErrorFields.RAW_MESSAGE.getName() : ErrorFields.RAW_MESSAGE.getName() + "_" + i; + // It's unclear if we need a rawMessageBytes field so commenting out for now + //String rawMessageBytesField = rawMessages.size() == 1 ? ErrorFields.RAW_MESSAGE_BYTES.getName() : ErrorFields.RAW_MESSAGE_BYTES.getName() + "_" + i; + if(rawMessage instanceof byte[]) { + errorMessage.put(rawMessageField, Bytes.toString((byte[])rawMessage)); + //errorMessage.put(rawMessageBytesField, com.google.common.primitives.Bytes.asList((byte[])rawMessage)); + } else if (rawMessage instanceof JSONObject) { + JSONObject rawMessageJSON = (JSONObject) rawMessage; + String rawMessageJSONString = rawMessageJSON.toJSONString(); + errorMessage.put(rawMessageField, rawMessageJSONString); + //errorMessage.put(rawMessageBytesField, com.google.common.primitives.Bytes.asList(rawMessageJSONString.getBytes(UTF_8))); + } else { + errorMessage.put(rawMessageField, rawMessage.toString()); + //errorMessage.put(rawMessageBytesField, com.google.common.primitives.Bytes.asList(rawMessage.toString().getBytes(UTF_8))); + } + } + } + } + + @SuppressWarnings({"unchecked"}) + private void addErrorHash(JSONObject errorMessage) { + if (rawMessages != null && rawMessages.size() == 1) { + Object rawMessage = rawMessages.get(0); + if (rawMessage instanceof JSONObject) { + JSONObject rawJSON = (JSONObject) rawMessage; + if (errorFields != null) { + errorMessage.put(ErrorFields.ERROR_FIELDS.getName(), String.join(",", errorFields)); + errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash(rawJSON, errorFields)); + } else { + errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash(rawJSON)); + } + } else if (rawMessage instanceof byte[]) { + errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash((byte[])rawMessage)); + } else { + errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash(rawMessage.toString().getBytes(UTF_8))); + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MetronError that = (MetronError) o; + + if (message != null ? !message.equals(that.message) : that.message != null) + return false; + if (throwable != null ? !throwable.equals(that.throwable) : that.throwable != null) + return false; + if (sensorType != null ? !sensorType.equals(that.sensorType) : that.sensorType != null) + return false; + if (errorType != null ? !errorType.equals(that.errorType) : that.errorType != null) + return false; + if (errorFields != null ? !errorFields.equals(that.errorFields) : that.errorFields != null) + return false; + return rawMessages != null ? rawMessages.equals(that.rawMessages) : that.rawMessages == null; + + } + + @Override + public int hashCode() { + int result = message != null ? message.hashCode() : 0; + result = 31 * result + (throwable != null ? throwable.hashCode() : 0); + result = 31 * result + (sensorType != null ? sensorType.hashCode() : 0); + result = 31 * result + (errorType != null ? errorType.hashCode() : 0); + result = 31 * result + (errorFields != null ? errorFields.hashCode() : 0); + result = 31 * result + (rawMessages != null ? rawMessages.hashCode() : 0); + return result; + } + +} diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/BytesFromPosition.java similarity index 67% rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java rename to metron-platform/metron-common/src/main/java/org/apache/metron/common/message/BytesFromPosition.java index fdd5fb8081..b73228f247 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/BytesFromPosition.java @@ -15,20 +15,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.metron.writer.message; +package org.apache.metron.common.message; import org.apache.storm.tuple.Tuple; -import org.json.simple.JSONObject; -public class NamedMessageGetter implements MessageGetter { - public static NamedMessageGetter DEFAULT = new NamedMessageGetter("message"); - private String messageName; - public NamedMessageGetter(String name) { - this.messageName = name; +public class BytesFromPosition implements MessageGetStrategy { + + private int position = 0; + + public BytesFromPosition() {}; + + public BytesFromPosition(int position) { + this.position = position; } + @Override - public JSONObject getMessage(Tuple tuple) { - return (JSONObject)tuple.getValueByField(messageName); + public byte[] get(Tuple tuple) { + return tuple.getBinary(position); } } diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/JSONFromField.java similarity index 70% rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java rename to metron-platform/metron-common/src/main/java/org/apache/metron/common/message/JSONFromField.java index da4a5495e1..39fe9dd6af 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/JSONFromField.java @@ -15,23 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.metron.writer.message; - +package org.apache.metron.common.message; import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; -public enum MessageGetters implements MessageGetter{ - RAW(RawMessageGetter.DEFAULT) - ,NAMED(NamedMessageGetter.DEFAULT) - ; - MessageGetter getter; - MessageGetters(MessageGetter getter) { - this.getter = getter; +public class JSONFromField implements MessageGetStrategy { + + private String fieldValue = "message"; + + public JSONFromField() {}; + + public JSONFromField(String fieldValue) { + this.fieldValue = fieldValue; } + @Override - public JSONObject getMessage(Tuple t) { - return getter.getMessage(t); + public JSONObject get(Tuple tuple) { + return (JSONObject) ((JSONObject) tuple.getValueByField(fieldValue)).clone(); } } diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/JSONFromPosition.java similarity index 70% rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java rename to metron-platform/metron-common/src/main/java/org/apache/metron/common/message/JSONFromPosition.java index 99a837869b..4407d4f709 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/JSONFromPosition.java @@ -15,34 +15,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.metron.writer.message; +package org.apache.metron.common.message; import org.apache.storm.tuple.Tuple; -import org.apache.metron.common.utils.JSONUtils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; -import java.io.UnsupportedEncodingException; +public class JSONFromPosition implements MessageGetStrategy { + + private int position = 0; -public class RawMessageGetter implements MessageGetter { - public static RawMessageGetter DEFAULT = new RawMessageGetter(0); private ThreadLocal parser = new ThreadLocal() { @Override protected JSONParser initialValue() { return new JSONParser(); } }; - int position = 0; - public RawMessageGetter(int position) { + + public JSONFromPosition() {}; + + public JSONFromPosition(int position) { this.position = position; } + @Override - public JSONObject getMessage(Tuple t) { - byte[] data = t.getBinary(position); + public JSONObject get(Tuple tuple) { try { - return (JSONObject) parser.get().parse(new String(data, "UTF8")); + return (JSONObject) parser.get().parse(new String(tuple.getBinary(position), "UTF8")); } catch (Exception e) { throw new IllegalStateException(e.getMessage(), e); } diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/MessageGetStrategy.java similarity index 85% rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java rename to metron-platform/metron-common/src/main/java/org/apache/metron/common/message/MessageGetStrategy.java index 99c825fa4f..0595ce1bfa 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/MessageGetStrategy.java @@ -15,12 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.metron.writer.message; +package org.apache.metron.common.message; import org.apache.storm.tuple.Tuple; -import org.json.simple.JSONObject; -public interface MessageGetter { - JSONObject getMessage(Tuple t); +public interface MessageGetStrategy { + + Object get(Tuple tuple); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/MessageGetters.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/MessageGetters.java new file mode 100644 index 0000000000..a496e08768 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/MessageGetters.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.message; + +import org.apache.metron.common.utils.ConversionUtils; + +import java.util.function.Function; + +/** + * MessageGetters is a convenience enum for looking up the various implementations of MessageGetStrategy. The MessageGetStrategy + * abstraction returns a value from a tuple. The implementations include: + *

    + *
  • BYTES_FROM_POSITION - gets a byte array from the provided position
    • + *
  • JSON_FROM_POSITION - gets a byte array from the provided position then converts to a string and parses the string to JSON
    • + *
  • JSON_FROM_FIELD - gets a JSONObject from the provided field
    • + *
  • OBJECT_FROM_FIELD - gets an Object from the provided field
    • + *
  • DEFAULT_BYTES_FROM_POSITION - gets a byte array from position 0
    • + *
  • DEFAULT_JSON_FROM_POSITION - gets a byte array from position 0 then converts to a string and parses the string to JSON
    • + *
  • DEFAULT_JSON_FROM_FIELD - gets a JSONObject from the "message" field
    • + *
  • DEFAULT_OBJECT_FROM_FIELD - gets an Object from the "message" field
    • + *
+ * + */ +public enum MessageGetters { + + BYTES_FROM_POSITION((String arg) -> new BytesFromPosition(ConversionUtils.convert(arg, Integer.class))), + JSON_FROM_POSITION((String arg) -> new JSONFromPosition(ConversionUtils.convert(arg, Integer.class))), + JSON_FROM_FIELD((String arg) -> new JSONFromField(arg)), + OBJECT_FROM_FIELD((String arg) -> new ObjectFromField(arg)), + DEFAULT_BYTES_FROM_POSITION(new BytesFromPosition()), + DEFAULT_JSON_FROM_POSITION(new JSONFromPosition()), + DEFAULT_JSON_FROM_FIELD(new JSONFromField()), + DEFAULT_OBJECT_FROM_FIELD(new ObjectFromField()); + + Function messageGetStrategyFunction; + MessageGetStrategy messageGetStrategy; + + MessageGetters(MessageGetStrategy messageGetStrategy) { + this.messageGetStrategy = messageGetStrategy; + } + + MessageGetters(Function messageGetStrategy) { + this.messageGetStrategyFunction = messageGetStrategy; + } + + public MessageGetStrategy get(String arg) { + return messageGetStrategyFunction.apply(arg); + } + + public MessageGetStrategy get() { + return messageGetStrategy; + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/ObjectFromField.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/ObjectFromField.java new file mode 100644 index 0000000000..120c09c921 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/ObjectFromField.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.message; + +import org.apache.storm.tuple.Tuple; + +public class ObjectFromField implements MessageGetStrategy { + + private String fieldValue = "message"; + + public ObjectFromField() {}; + + public ObjectFromField(String fieldValue) { + this.fieldValue = fieldValue; + } + + @Override + public Object get(Tuple tuple) { + return tuple.getValueByField(fieldValue); + } +} diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java index 3b3f42683f..f4e3a8d01e 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java @@ -17,30 +17,17 @@ */ package org.apache.metron.common.utils; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.tuple.Values; -import org.apache.commons.beanutils.Converter; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.commons.lang3.tuple.Pair; import org.apache.metron.common.Constants; -import org.json.simple.JSONObject; +import org.apache.metron.common.error.MetronError; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Values; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.Optional; import java.util.function.Function; @@ -102,74 +89,16 @@ private static String formatReason(String reason, Optional t) { } } - @SuppressWarnings("unchecked") - public static JSONObject generateErrorMessage(String message, Throwable t) + public static void handleError(OutputCollector collector, MetronError error) { - return generateErrorMessage(message, t, Optional.empty(), Optional.empty()); - } - public static JSONObject generateErrorMessage(String message - , Throwable t - , Optional sensorType - , Optional rawMessage - ) - { - JSONObject error_message = new JSONObject(); - - /* - * Save full stack trace in object. - */ - String stackTrace = ExceptionUtils.getStackTrace(t); - - String exception = t.toString(); - - - error_message.put("time", System.currentTimeMillis()); - try { - error_message.put("hostname", InetAddress.getLocalHost().getHostName()); - } catch (UnknownHostException ex) { - + collector.emit(Constants.ERROR_STREAM, new Values(error.getJSONObject())); + Optional throwable = error.getThrowable(); + if (throwable.isPresent()) { + collector.reportError(throwable.get()); } - if(rawMessage.isPresent()) { - if(rawMessage.get() instanceof byte[]) { - error_message.put("rawMessage", Bytes.toString((byte[])rawMessage.get())); - error_message.put("rawMessage_bytes", toByteArrayList((byte[])rawMessage.get())); - } - else { - error_message.put("rawMessage", rawMessage.get()); - } - } - error_message.put("message", message); - error_message.put(Constants.SENSOR_TYPE, StringUtils.join("_", sensorType, Optional.of("error"))); - error_message.put("exception", exception); - error_message.put("stack", stackTrace); - return error_message; - } - - private static List toByteArrayList(byte[] list) { - List ret = new ArrayList<>(); - for(byte b : list) { - ret.add(b); - } - return ret; } - public static void handleError(OutputCollector collector, Throwable t, String errorStream) { - handleError(collector, t, errorStream, Optional.empty(), Optional.empty()); - } - public static void handleError(OutputCollector collector - , Throwable t - , String errorStream - , Optional sensorType - , Optional rawMessage - ) - { - JSONObject error = ErrorUtils.generateErrorMessage(t.getMessage(), t, sensorType, rawMessage); - collector.emit(errorStream, new Values(error)); - collector.reportError(t); - } - - public static String generateThreadDump() { final StringBuilder dump = new StringBuilder(); final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/HashUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/HashUtils.java new file mode 100644 index 0000000000..b5170cef57 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/HashUtils.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.utils; + +import org.apache.commons.codec.digest.DigestUtils; +import org.json.simple.JSONObject; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class HashUtils { + + public static String getMessageHash(JSONObject message, Collection hashFields) { + List hashElements = hashFields.stream().map(errorField -> + String.format("%s-%s", errorField, message.get(errorField))).collect(Collectors.toList()); + return DigestUtils.sha256Hex(String.join("|", hashElements).getBytes(UTF_8)); + } + + public static String getMessageHash(JSONObject message) { + return DigestUtils.sha256Hex(message.toJSONString().getBytes(UTF_8)); + } + + public static String getMessageHash(byte[] bytes) { + return DigestUtils.sha256Hex(bytes); + } +} diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java new file mode 100644 index 0000000000..5e505a8579 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.error; + +import com.google.common.collect.Sets; +import com.google.common.primitives.Bytes; +import org.apache.metron.common.Constants; +import org.json.simple.JSONObject; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + +import static org.apache.metron.common.Constants.ErrorFields.RAW_MESSAGE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MetronErrorTest { + + private JSONObject message1 = new JSONObject(); + private JSONObject message2 = new JSONObject(); + + @Before + public void setup() { + message1.put("value", "message1"); + message2.put("value", "message2"); + } + + @Test + public void getJSONObjectShouldReturnBasicInformation() { + MetronError error = new MetronError() + .withMessage("test message") + .withErrorType(Constants.ErrorType.PARSER_ERROR) + .withSensorType("sensorType"); + + JSONObject errorJSON = error.getJSONObject(); + assertEquals("test message", errorJSON.get(Constants.ErrorFields.MESSAGE.getName())); + assertEquals(Constants.ErrorType.PARSER_ERROR.getType(), errorJSON.get(Constants.ErrorFields.ERROR_TYPE.getName())); + assertEquals("error", errorJSON.get(Constants.SENSOR_TYPE)); + assertEquals("sensorType", errorJSON.get(Constants.ErrorFields.FAILED_SENSOR_TYPE.getName())); + assertTrue(((String) errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())).length() > 0); + assertTrue(((long) errorJSON.get(Constants.ErrorFields.TIMESTAMP.getName())) > 0); + } + + @Test + public void getJSONObjectShouldHandleThrowable() { + Throwable e = new Exception("test exception"); + MetronError error = new MetronError().withThrowable(e); + + JSONObject errorJSON = error.getJSONObject(); + assertEquals("java.lang.Exception: test exception", errorJSON.get(Constants.ErrorFields.EXCEPTION.getName())); + assertTrue(((String) errorJSON.get(Constants.ErrorFields.STACK.getName())).startsWith("java.lang.Exception: test exception")); + assertEquals(e.getMessage(), errorJSON.get(Constants.ErrorFields.MESSAGE.getName())); + } + + @Test + public void getJSONObjectShouldIncludeRawMessages() { + JSONObject message1 = new JSONObject(); + JSONObject message2 = new JSONObject(); + message1.put("value", "message1"); + message2.put("value", "message2"); + MetronError error = new MetronError().withRawMessages(Arrays.asList(message1, message2)); + + JSONObject errorJSON = error.getJSONObject(); + + assertEquals("{\"value\":\"message1\"}", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName() + "_0")); + assertEquals("{\"value\":\"message2\"}", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName() + "_1")); + + error = new MetronError().addRawMessage("raw message".getBytes()); + errorJSON = error.getJSONObject(); + assertEquals("raw message", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName())); + // It's unclear if we need a rawMessageBytes field so commenting out for now + //assertEquals(Bytes.asList("raw message".getBytes()), errorJSON.get(Constants.ErrorFields.RAW_MESSAGE_BYTES.getName())); + assertEquals("3b02cb29676bc448c69da1ec5eef7c89f4d6dc6a5a7ce0296ea25b207eea36be", errorJSON.get(Constants.ErrorFields.ERROR_HASH.getName())); + + error = new MetronError().addRawMessage(message1); + errorJSON = error.getJSONObject(); + assertEquals("{\"value\":\"message1\"}", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName())); + assertEquals("e8aaf87c8494d345aac2d612ffd94fcf0b98c975fe6c4b991e2f8280a3a0bd10", errorJSON.get(Constants.ErrorFields.ERROR_HASH.getName())); + } + + @Test + public void getJSONObjectShouldIncludeErrorFields() { + JSONObject message = new JSONObject(); + message.put("field1", "value1"); + message.put("field2", "value2"); + + MetronError error = new MetronError().addRawMessage(message).withErrorFields(Sets.newHashSet("field1", "field2")); + + JSONObject errorJSON = error.getJSONObject(); + assertEquals(Sets.newHashSet("field1", "field2"), Sets.newHashSet(((String) errorJSON.get(Constants.ErrorFields.ERROR_FIELDS.getName())).split(","))); + assertEquals("04a2629c39e098c3944be85f35c75876598f2b44b8e5e3f52c59fa1ac182817c", errorJSON.get(Constants.ErrorFields.ERROR_HASH.getName())); + } +} diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/message/MessageGettersTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/message/MessageGettersTest.java new file mode 100644 index 0000000000..ea7583ae07 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/message/MessageGettersTest.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.message; + +import org.apache.storm.tuple.Tuple; +import org.json.simple.JSONObject; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MessageGettersTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void bytesFromPositionShouldReturnBytes() { + Tuple tuple = mock(Tuple.class); + when(tuple.getBinary(1)).thenReturn("bytes".getBytes(UTF_8)); + + MessageGetStrategy messageGetStrategy = MessageGetters.BYTES_FROM_POSITION.get("1"); + assertEquals("bytes", new String((byte[]) messageGetStrategy.get(tuple), UTF_8)); + } + + @Test + public void jsonFromPositionShouldReturnJSON() { + Tuple tuple = mock(Tuple.class); + when(tuple.getBinary(1)).thenReturn("{\"field\":\"value\"}".getBytes(UTF_8)); + + JSONObject expected = new JSONObject(); + expected.put("field", "value"); + MessageGetStrategy messageGetStrategy = MessageGetters.JSON_FROM_POSITION.get("1"); + assertEquals(expected, messageGetStrategy.get(tuple)); + } + + @Test + public void jsonFromPositionShouldThrowException() { + exception.expect(IllegalStateException.class); + + Tuple tuple = mock(Tuple.class); + when(tuple.getBinary(1)).thenReturn("{\"field\":".getBytes(UTF_8)); + + MessageGetStrategy messageGetStrategy = MessageGetters.JSON_FROM_POSITION.get("1"); + messageGetStrategy.get(tuple); + } + + @Test + public void jsonFromFieldShouldReturnJSON() { + JSONObject actual = new JSONObject(); + actual.put("field", "value"); + Tuple tuple = mock(Tuple.class); + when(tuple.getValueByField("tuple_field")).thenReturn(actual); + + JSONObject expected = new JSONObject(); + expected.put("field", "value"); + MessageGetStrategy messageGetStrategy = MessageGetters.JSON_FROM_FIELD.get("tuple_field"); + assertEquals(expected, messageGetStrategy.get(tuple)); + } + + @Test + public void objectFromFieldShouldReturnObject() { + Object actual = "object"; + Tuple tuple = mock(Tuple.class); + when(tuple.getValueByField("tuple_field")).thenReturn(actual); + + Object expected = "object"; + MessageGetStrategy messageGetStrategy = MessageGetters.OBJECT_FROM_FIELD.get("tuple_field"); + assertEquals(expected, messageGetStrategy.get(tuple)); + } + + @Test + public void defaultBytesFromPositionShouldReturnBytes() { + Tuple tuple = mock(Tuple.class); + when(tuple.getBinary(0)).thenReturn("bytes".getBytes(UTF_8)); + + MessageGetStrategy messageGetStrategy = MessageGetters.DEFAULT_BYTES_FROM_POSITION.get(); + assertEquals("bytes", new String((byte[]) messageGetStrategy.get(tuple), UTF_8)); + } + + @Test + public void defaultJSONFromPositionShouldReturnJSON() { + Tuple tuple = mock(Tuple.class); + when(tuple.getBinary(0)).thenReturn("{\"field\":\"value\"}".getBytes(UTF_8)); + + JSONObject expected = new JSONObject(); + expected.put("field", "value"); + MessageGetStrategy messageGetStrategy = MessageGetters.DEFAULT_JSON_FROM_POSITION.get(); + assertEquals(expected, messageGetStrategy.get(tuple)); + } + + @Test + public void defaultJSONFromFieldShouldReturnJSON() { + JSONObject actual = new JSONObject(); + actual.put("field", "value"); + Tuple tuple = mock(Tuple.class); + when(tuple.getValueByField("message")).thenReturn(actual); + + JSONObject expected = new JSONObject(); + expected.put("field", "value"); + MessageGetStrategy messageGetStrategy = MessageGetters.DEFAULT_JSON_FROM_FIELD.get(); + assertEquals(expected, messageGetStrategy.get(tuple)); + } + + @Test + public void defaultObjectFromFieldShouldReturnObject() { + Object actual = "object"; + Tuple tuple = mock(Tuple.class); + when(tuple.getValueByField("message")).thenReturn(actual); + + Object expected = "object"; + MessageGetStrategy messageGetStrategy = MessageGetters.DEFAULT_OBJECT_FROM_FIELD.get(); + assertEquals(expected, messageGetStrategy.get(tuple)); + } +} diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/ErrorUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/ErrorUtilsTest.java index f11a5c9842..77ea9da197 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/ErrorUtilsTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/ErrorUtilsTest.java @@ -17,6 +17,10 @@ */ package org.apache.metron.common.utils; +import org.apache.metron.common.Constants; +import org.apache.metron.common.error.MetronError; +import org.apache.metron.test.error.MetronErrorJSONMatcher; +import org.apache.storm.task.OutputCollector; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -25,6 +29,12 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.nullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; public class ErrorUtilsTest { @@ -62,4 +72,15 @@ public void illegal_state_throws_exception_with_reason_and_cause() throws Except exception.expectCause(instanceOf(IOException.class)); ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("illegal state happened", new IOException("bad io")); } + + @Test + public void handleErrorShouldEmitAndReportError() throws Exception { + Throwable e = new Exception("error"); + MetronError error = new MetronError().withMessage("error message").withThrowable(e); + OutputCollector collector = mock(OutputCollector.class); + + ErrorUtils.handleError(collector, error); + verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); + verify(collector, times(1)).reportError(any()); + } } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/HashUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/HashUtilsTest.java new file mode 100644 index 0000000000..303734101f --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/HashUtilsTest.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.utils; + +import org.json.simple.JSONObject; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; + +@SuppressWarnings({"unchecked"}) +public class HashUtilsTest { + + @Test + public void getMessageHashShouldReturnHashForHashFields() { + JSONObject message = new JSONObject(); + message.put("field1", "value1"); + message.put("field2", "value2"); + message.put("field3", "value3"); + Collection fields = Arrays.asList("field2", "field3"); + assertEquals("6eab1c2c827387803ce457c76552f0511858fc1f9505c7dc620e198c0d1f4d02", HashUtils.getMessageHash(message, fields)); + } + + @Test + public void getMessageHashShouldReturnHashForMessage() { + JSONObject message = new JSONObject(); + message.put("field1", "value1"); + message.put("field2", "value2"); + message.put("field3", "value3"); + assertEquals("a76cdafc5aa49180c0b22c78d4415c505f9997c54847cec6c623f4cacf6a2811", HashUtils.getMessageHash(message)); + } + + @Test + public void getMessageHashShouldReturnHashForBytes() { + assertEquals("ab530a13e45914982b79f9b7e3fba994cfd1f3fb22f71cea1afbf02b460c6d1d", HashUtils.getMessageHash("message".getBytes(UTF_8))); + } +} diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties index c2c10af059..27e9173a33 100644 --- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties +++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties @@ -26,7 +26,7 @@ kafka.start=WHERE_I_LEFT_OFF ##### Indexing ##### index.input.topic=indexing -index.error.topic=indexing_error +index.error.topic=indexing writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter ##### ElasticSearch ##### diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java index acc1565e5c..87c0081bfd 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java @@ -17,8 +17,8 @@ */ package org.apache.metron.elasticsearch.integration; -import org.apache.metron.common.Constants; import org.apache.metron.common.interfaces.FieldNameConverter; +import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; import org.apache.metron.elasticsearch.writer.ElasticsearchFieldNameConverter; import org.apache.metron.indexing.integration.IndexingIntegrationTest; import org.apache.metron.integration.ComponentRunner; @@ -26,7 +26,6 @@ import org.apache.metron.integration.Processor; import org.apache.metron.integration.ProcessorResult; import org.apache.metron.integration.ReadinessState; -import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; import org.apache.metron.integration.components.KafkaComponent; import java.io.File; @@ -76,7 +75,7 @@ public ReadinessState process(ComponentRunner runner) { throw new IllegalStateException("Unable to retrieve indexed documents.", e); } if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) { - errors = kafkaComponent.readMessages(Constants.INDEXING_ERROR_TOPIC); + errors = kafkaComponent.readMessages(ERROR_TOPIC); if(errors.size() > 0){ return ReadinessState.READY; } diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/src/main/config/enrichment.properties index 84d8461ee4..c905d30ed0 100644 --- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties +++ b/metron-platform/metron-enrichment/src/main/config/enrichment.properties @@ -20,8 +20,8 @@ kafka.zk=node1:2181 kafka.broker=node1:6667 enrichment.output.topic=indexing -enrichment.error.topic=enrichments_error -threat.intel.error.topic=threatintel_error +enrichment.error.topic=indexing +threat.intel.error.topic=indexing ##### Metrics ##### @@ -71,7 +71,11 @@ bolt.hbase.partitioner.region.info.refresh.interval.mins=60 ##### Threat Intel ##### -threat.intel.tracker.table= -threat.intel.tracker.cf= +threat.intel.tracker.table=access_tracker +threat.intel.tracker.cf=t threat.intel.ip.table= threat.intel.ip.cf= +threat.intel.simple.hbase.table=threatintel +threat.intel.simple.hbase.cf=t +enrichment.simple.hbase.table=enrichment +enrichment.simple.hbase.cf=t diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java index bb77b84ff5..d6f1304ed9 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java @@ -18,6 +18,7 @@ package org.apache.metron.enrichment.bolt; +import org.apache.metron.common.error.MetronError; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; @@ -41,7 +42,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; /** @@ -225,6 +228,12 @@ public void execute(Tuple tuple) { catch(Exception e) { LOG.error(e.getMessage(), e); error = true; + MetronError metronError = new MetronError() + .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) + .withThrowable(e) + .withErrorFields(new HashSet() {{ add(field); }}) + .addRawMessage(rawMessage); + ErrorUtils.handleError(collector, metronError); continue; } } @@ -257,11 +266,14 @@ public void execute(Tuple tuple) { // errors, so this is made available in order to ensure ERROR_STREAM is output properly. protected void handleError(String key, JSONObject rawMessage, String subGroup, JSONObject enrichedMessage, Exception e) { LOG.error("[Metron] Unable to enrich message: " + rawMessage, e); - JSONObject error = ErrorUtils.generateErrorMessage("Enrichment problem: " + rawMessage, e); if (key != null) { collector.emit(enrichmentType, new Values(key, enrichedMessage, subGroup)); } - collector.emit(ERROR_STREAM, new Values(error)); + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) + .withThrowable(e) + .addRawMessage(rawMessage); + ErrorUtils.handleError(collector, error); } @Override diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java index 101d056a9b..3bbb3f50fd 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java @@ -17,27 +17,29 @@ */ package org.apache.metron.enrichment.bolt; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; import com.google.common.base.Joiner; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.Sets; -import org.apache.metron.common.bolt.ConfiguredBolt; -import org.apache.metron.common.utils.ErrorUtils; -import org.json.simple.JSONObject; +import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt; +import org.apache.metron.common.error.MetronError; +import org.apache.metron.common.message.MessageGetStrategy; +import org.apache.metron.common.message.MessageGetters; +import org.apache.metron.common.utils.ErrorUtils; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.util.HashMap; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; public abstract class JoinBolt extends ConfiguredEnrichmentBolt { @@ -48,6 +50,9 @@ public abstract class JoinBolt extends ConfiguredEnrichmentBolt { protected transient CacheLoader> loader; protected transient LoadingCache> cache; + private transient MessageGetStrategy keyGetStrategy; + private transient MessageGetStrategy subgroupGetStrategy; + private transient MessageGetStrategy messageGetStrategy; protected Long maxCacheSize; protected Long maxTimeRetain; @@ -68,6 +73,9 @@ public JoinBolt withMaxTimeRetain(long maxTimeRetain) { @Override public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { super.prepare(map, topologyContext, outputCollector); + keyGetStrategy = MessageGetters.OBJECT_FROM_FIELD.get("key"); + subgroupGetStrategy = MessageGetters.OBJECT_FROM_FIELD.get("subgroup"); + messageGetStrategy = MessageGetters.OBJECT_FROM_FIELD.get("message"); this.collector = outputCollector; if (this.maxCacheSize == null) { throw new IllegalStateException("maxCacheSize must be specified"); @@ -91,10 +99,10 @@ public Map load(String key) throws Exception { @Override public void execute(Tuple tuple) { String streamId = tuple.getSourceStreamId(); - String key = (String) tuple.getValueByField("key"); - String subgroup = (String) tuple.getValueByField("subgroup"); + String key = (String) keyGetStrategy.get(tuple); + String subgroup = (String) subgroupGetStrategy.get(tuple); streamId = Joiner.on(":").join("" + streamId, subgroup == null?"":subgroup); - V message = (V) tuple.getValueByField("message"); + V message = (V) messageGetStrategy.get(tuple); try { Map streamMessageMap = cache.get(key); if (streamMessageMap.containsKey(streamId)) { @@ -127,10 +135,13 @@ public void execute(Tuple tuple) { } } catch (Exception e) { LOG.error("[Metron] Unable to join messages: " + message, e); - JSONObject error = ErrorUtils.generateErrorMessage("Joining problem: " + message, e); + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) + .withMessage("Joining problem: " + message) + .withThrowable(e) + .addRawMessage(message); + ErrorUtils.handleError(collector, error); collector.ack(tuple); - collector.emit("error", new Values(error)); - collector.reportError(e); } } diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java index c5c1294f47..1f33060ebf 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java @@ -18,6 +18,8 @@ package org.apache.metron.enrichment.bolt; import org.apache.log4j.Level; +import org.apache.metron.common.message.MessageGetStrategy; +import org.apache.metron.common.message.MessageGetters; import org.apache.metron.common.writer.BulkWriterResponse; import org.apache.metron.test.utils.UnitTestHelper; import org.apache.metron.writer.BulkWriterComponent; @@ -112,9 +114,13 @@ public void parseMessages() throws ParseException { @Mock private BulkMessageWriter bulkMessageWriter; + @Mock + private MessageGetStrategy messageGetStrategy; + @Test public void test() throws Exception { - BulkMessageWriterBolt bulkMessageWriterBolt = new BulkMessageWriterBolt("zookeeperUrl").withBulkMessageWriter(bulkMessageWriter); + BulkMessageWriterBolt bulkMessageWriterBolt = new BulkMessageWriterBolt("zookeeperUrl") + .withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()).withMessageGetterField("message"); bulkMessageWriterBolt.setCuratorFramework(client); bulkMessageWriterBolt.setTreeCache(cache); bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType, new FileInputStream(sampleSensorIndexingConfigPath)); diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java index b0076a4d19..90322fe172 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java @@ -17,18 +17,22 @@ */ package org.apache.metron.enrichment.bolt; -import org.apache.log4j.Level; -import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; -import org.apache.metron.test.utils.UnitTestHelper; -import org.apache.storm.tuple.Values; +import com.google.common.cache.CacheLoader; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; +import org.apache.log4j.Level; import org.apache.metron.TestConstants; -import org.apache.metron.test.bolt.BaseEnrichmentBoltTest; -import org.apache.metron.enrichment.configuration.Enrichment; +import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.error.MetronError; +import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; +import org.apache.metron.enrichment.configuration.Enrichment; import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; -import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.test.bolt.BaseEnrichmentBoltTest; +import org.apache.metron.test.error.MetronErrorJSONMatcher; +import org.apache.metron.test.utils.UnitTestHelper; +import org.apache.storm.tuple.Values; import org.hamcrest.Description; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -43,12 +47,16 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class GenericEnrichmentBoltTest extends BaseEnrichmentBoltTest { @@ -194,7 +202,10 @@ protected void initializeStellar() { UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.FATAL); genericEnrichmentBolt.execute(tuple); UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.ERROR); - verify(outputCollector, times(1)).emit(eq("error"), any(Values.class)); + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) + .withThrowable(new Exception("Could not parse binary stream to JSON")); + verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); when(tuple.getStringByField("key")).thenReturn(key); when(tuple.getValueByField("message")).thenReturn(originalMessage); when(enrichmentAdapter.enrich(any())).thenReturn(new JSONObject()); @@ -217,6 +228,19 @@ protected void initializeStellar() { verify(enrichmentAdapter, times(1)).logAccess(cacheKey2); verify(outputCollector, times(1)).emit(eq(enrichmentType), argThat(new EnrichedMessageMatcher(key, enrichedMessage))); - + reset(outputCollector); + genericEnrichmentBolt.cache.invalidateAll(); + when(enrichmentAdapter.enrich(cacheKey1)).thenReturn(null); + genericEnrichmentBolt.execute(tuple); + error = new MetronError() + .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) + .withErrorFields(new HashSet() {{ add("field1"); }}) + .addRawMessage(new JSONObject() {{ + put("field1", "value1"); + put("field2", "value2"); + put("source.type", "test"); + }}) + .withThrowable(new CacheLoader.InvalidCacheLoadException("CacheLoader returned null for key CacheKey{field='field1', value='value1'}.")); + verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); } } diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java index 5b06d336fa..9f12fcd858 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java @@ -17,10 +17,14 @@ */ package org.apache.metron.enrichment.bolt; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Values; +import com.google.common.cache.LoadingCache; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.Constants; +import org.apache.metron.common.error.MetronError; import org.apache.metron.test.bolt.BaseEnrichmentBoltTest; +import org.apache.metron.test.error.MetronErrorJSONMatcher; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.tuple.Values; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -31,11 +35,13 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -89,7 +95,7 @@ public void parseMessages() { } @Test - public void test() { + public void test() throws Exception { StandAloneJoinBolt joinBolt = new StandAloneJoinBolt("zookeeperUrl"); joinBolt.setCuratorFramework(client); joinBolt.setTreeCache(cache); @@ -126,5 +132,16 @@ public void test() { joinBolt.execute(tuple); verify(outputCollector, times(1)).emit(eq("message"), any(tuple.getClass()), eq(new Values(key, joinedMessage))); verify(outputCollector, times(1)).ack(tuple); + + joinBolt.cache = mock(LoadingCache.class); + when(joinBolt.cache.get(key)).thenThrow(new ExecutionException(new Exception("join exception"))); + joinBolt.execute(tuple); + + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) + .withMessage("Joining problem: {}") + .withThrowable(new ExecutionException(new Exception("join exception"))) + .addRawMessage(new JSONObject()); + verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); } } diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index 7306c64294..e012c556bd 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -76,6 +76,7 @@ import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.*; public class EnrichmentIntegrationTest extends BaseIntegrationTest { + private static final String ERROR_TOPIC = "enrichment_error"; private static final String SRC_IP = "ip_src_addr"; private static final String DST_IP = "ip_dst_addr"; private static final String MALICIOUS_IP_TYPE = "malicious_ip"; @@ -139,13 +140,13 @@ public void test() throws Exception { setProperty("enrichment.simple.hbase.table", enrichmentsTableName); setProperty("enrichment.simple.hbase.cf", cf); setProperty("enrichment.output.topic", Constants.INDEXING_TOPIC); - setProperty("enrichment.error.topic", Constants.ENRICHMENT_ERROR_TOPIC); + setProperty("enrichment.error.topic", ERROR_TOPIC); }}; final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties); final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList() {{ add(new KafkaComponent.Topic(Constants.ENRICHMENT_TOPIC, 1)); add(new KafkaComponent.Topic(Constants.INDEXING_TOPIC, 1)); - add(new KafkaComponent.Topic(Constants.ENRICHMENT_ERROR_TOPIC, 1)); + add(new KafkaComponent.Topic(ERROR_TOPIC, 1)); }}); String globalConfigStr = null; { @@ -201,15 +202,14 @@ public void test() throws Exception { fluxComponent.submitTopology(); kafkaComponent.writeMessages(Constants.ENRICHMENT_TOPIC, inputMessages); - ProcessorResult>> result = runner.process(getProcessor()); - // We expect failures, so we don't care if result returned failure or not - List> docs = result.getResult(); + ProcessorResult>>> result = runner.process(getProcessor()); + Map>> outputMessages = result.getResult(); + List> docs = outputMessages.get(Constants.INDEXING_TOPIC); Assert.assertEquals(inputMessages.size(), docs.size()); validateAll(docs); - - List errors = result.getProcessErrors(); + List> errors = outputMessages.get(ERROR_TOPIC); Assert.assertEquals(inputMessages.size(), errors.size()); - validateErrors(result.getProcessErrors()); + validateErrors(errors); } finally { runner.stop(); } @@ -234,10 +234,12 @@ public static void validateAll(List> docs) { } } - protected void validateErrors(List errors) { - for(byte[] error : errors) { - // Don't reconstruct the entire message, just ensure it contains the known error message inside. - Assert.assertTrue(new String(error).contains(ErrorEnrichmentBolt.TEST_ERROR_MESSAGE)); + protected void validateErrors(List> errors) { + for(Map error : errors) { + Assert.assertEquals("Test throwing error from ErrorEnrichmentBolt", error.get(Constants.ErrorFields.MESSAGE.getName())); + Assert.assertEquals("java.lang.IllegalStateException: Test throwing error from ErrorEnrichmentBolt", error.get(Constants.ErrorFields.EXCEPTION.getName())); + Assert.assertEquals(Constants.ErrorType.ENRICHMENT_ERROR.getType(), error.get(Constants.ErrorFields.ERROR_TYPE.getName())); + Assert.assertEquals("{\"rawMessage\":\"Error Test Raw Message String\"}", error.get(Constants.ErrorFields.RAW_MESSAGE.getName())); } } @@ -504,39 +506,47 @@ private static Set setOf(String... items) { return ret; } + private static List> loadMessages(List outputMessages) { + List> tmp = new ArrayList<>(); + Iterables.addAll(tmp + , Iterables.transform(outputMessages + , message -> { + try { + return new HashMap<>(JSONUtils.INSTANCE.load(new String(message) + , new TypeReference>() {} + ) + ); + } catch (Exception ex) { + throw new IllegalStateException(ex); + } + } + ) + ); + return tmp; + } @SuppressWarnings("unchecked") - private Processor>> getProcessor() { + private KafkaProcessor>>> getProcessor(){ - KafkaProcessor>> kafkaProcessor = new KafkaProcessor<>().withKafkaComponentName("kafka") + return new KafkaProcessor<>() + .withKafkaComponentName("kafka") .withReadTopic(Constants.INDEXING_TOPIC) - .withErrorTopic(Constants.ENRICHMENT_ERROR_TOPIC) - .withInvalidTopic(Constants.INVALID_STREAM) + .withErrorTopic(ERROR_TOPIC) .withValidateReadMessages(new Function() { @Nullable @Override public Boolean apply(@Nullable KafkaMessageSet messageSet) { - // this test is written to return 10 errors and 10 messages - // we can just check when the messages match here - // if they do then we are good - return messageSet.getMessages().size() == inputMessages.size(); + return (messageSet.getMessages().size() == inputMessages.size()) && (messageSet.getErrors().size() == inputMessages.size()); } }) - .withProvideResult(new Function>>() { + .withProvideResult(new Function>>>(){ @Nullable @Override - public List> apply(@Nullable KafkaMessageSet messageSet) { - List> docs = new ArrayList<>(); - for (byte[] message : messageSet.getMessages()) { - try { - docs.add(JSONUtils.INSTANCE.load(new String(message), new TypeReference>() { - })); - } catch (IOException e) { - throw new IllegalStateException(e.getMessage(), e); - } - } - return docs; + public Map>> apply(@Nullable KafkaMessageSet messageSet) { + return new HashMap>>() {{ + put(Constants.INDEXING_TOPIC, loadMessages(messageSet.getMessages())); + put(ERROR_TOPIC, loadMessages(messageSet.getErrors())); + }}; } }); - return kafkaProcessor; } } diff --git a/metron-platform/metron-indexing/README.md b/metron-platform/metron-indexing/README.md index 5296ea0a05..f243eefd3d 100644 --- a/metron-platform/metron-indexing/README.md +++ b/metron-platform/metron-indexing/README.md @@ -24,7 +24,7 @@ and sent to * An indexing bolt configured to write to either elasticsearch or Solr * An indexing bolt configured to write to HDFS under `/apps/metron/enrichment/indexed` -Errors during indexing are sent to a kafka queue called `index_errors` +By default, errors during indexing are sent back into the `indexing` kafka queue so that they can be indexed and archived. ##Sensor Indexing Configuration The sensor specific configuration is intended to configure the diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/error.json b/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/error.json new file mode 100644 index 0000000000..4c9d786223 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/error.json @@ -0,0 +1,17 @@ +{ + "hdfs" : { + "index": "error", + "batchSize": 5, + "enabled" : true + }, + "elasticsearch" : { + "index": "error", + "batchSize": 5, + "enabled" : true + }, + "solr" : { + "index": "error", + "batchSize": 5, + "enabled" : true + } +} diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml index 8bf8f48a40..3e329f4008 100644 --- a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml +++ b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml @@ -101,7 +101,8 @@ bolts: - ref: "indexWriter" - name: "withMessageGetter" args: - - "RAW" + - "DEFAULT_JSON_FROM_POSITION" + - id: "hdfsIndexingBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -112,7 +113,7 @@ bolts: - ref: "hdfsWriter" - name: "withMessageGetter" args: - - "RAW" + - "DEFAULT_JSON_FROM_POSITION" - id: "indexingErrorBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java index ae04e43c59..60cd1d1c6a 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java @@ -44,7 +44,6 @@ import org.junit.Test; import javax.annotation.Nullable; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.util.*; @@ -53,6 +52,7 @@ import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient; public abstract class IndexingIntegrationTest extends BaseIntegrationTest { + protected static final String ERROR_TOPIC = "indexing_error"; protected String hdfsDir = "target/indexingIntegrationTest/hdfs"; protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed"; protected String fluxPath = "../metron-indexing/src/main/flux/indexing/remote.yaml"; @@ -125,7 +125,7 @@ public void test() throws Exception { setProperty("indexing.workers", "1"); setProperty("indexing.executors", "0"); setProperty("index.input.topic", Constants.INDEXING_TOPIC); - setProperty("index.error.topic", Constants.INDEXING_ERROR_TOPIC); + setProperty("index.error.topic", ERROR_TOPIC); setProperty("index.date.format", dateFormat); //HDFS settings @@ -138,7 +138,7 @@ public void test() throws Exception { final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties); final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList() {{ add(new KafkaComponent.Topic(Constants.INDEXING_TOPIC, 1)); - add(new KafkaComponent.Topic(Constants.INDEXING_ERROR_TOPIC, 1)); + add(new KafkaComponent.Topic(ERROR_TOPIC, 1)); }}); List> inputDocs = new ArrayList<>(); for(byte[] b : inputMessages) { diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/ProcessorResult.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/ProcessorResult.java index 3eb4e8f074..c3096863d4 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/ProcessorResult.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/ProcessorResult.java @@ -24,7 +24,6 @@ public class ProcessorResult { public static class Builder{ T result; List processErrors; - List processInvalids; public Builder(){} @@ -38,25 +37,18 @@ public Builder withProcessErrors(List processErrors){ return this; } - public Builder withProcessInvalids(List processInvalids){ - this.processInvalids = processInvalids; - return this; - } - public ProcessorResult build(){ - return new ProcessorResult(result,processErrors,processInvalids); + return new ProcessorResult(result,processErrors); } } T result; List processErrors; - List processInvalids; @SuppressWarnings("unchecked") - public ProcessorResult(T result,List processErrors, List processInvalids){ + public ProcessorResult(T result,List processErrors){ this.result = result; this.processErrors = processErrors == null ? new ArrayList() : processErrors; - this.processInvalids = processInvalids == null ? new ArrayList() : processInvalids; } public T getResult(){ @@ -67,12 +59,8 @@ public List getProcessErrors(){ return processErrors; } - public List getProcessInvalids(){ - return processInvalids; - } - public boolean failed(){ - return processErrors.size() > 0 || processInvalids.size() > 0; + return processErrors.size() > 0; } public void getBadResults(StringBuffer buffer){ @@ -84,10 +72,5 @@ public void getBadResults(StringBuffer buffer){ buffer.append(new String(outputMessage)); } buffer.append("\n"); - buffer.append(String.format("%d Invalid Messages", processInvalids.size())); - for (byte[] outputMessage : processInvalids) { - buffer.append(new String(outputMessage)); - } - buffer.append("\n"); } } diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/processors/KafkaMessageSet.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/processors/KafkaMessageSet.java index 4227933f8c..683fe6a92f 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/processors/KafkaMessageSet.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/processors/KafkaMessageSet.java @@ -23,12 +23,10 @@ public class KafkaMessageSet{ public List messages; public List errors; - public List invalids; - public KafkaMessageSet(List messages, List errors, List invalids) { + public KafkaMessageSet(List messages, List errors) { this.messages = messages; this.errors = errors; - this.invalids = invalids; } @@ -38,7 +36,4 @@ public List getMessages() { public List getErrors() { return errors; } - public List getInvalids() { - return invalids; - } } diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/processors/KafkaProcessor.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/processors/KafkaProcessor.java index 6fdbbf4a73..63f073ddbe 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/processors/KafkaProcessor.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/processors/KafkaProcessor.java @@ -30,10 +30,8 @@ public class KafkaProcessor implements Processor { private String kafkaComponentName; private String readTopic; private String errorTopic; - private String invalidTopic; private List messages = new LinkedList<>(); private List errors = new LinkedList<>(); - private List invalids = new LinkedList<>(); public KafkaProcessor(){} public KafkaProcessor withKafkaComponentName(String name){ @@ -48,10 +46,6 @@ public KafkaProcessor withErrorTopic(String topicName){ this.errorTopic = topicName; return this; } - public KafkaProcessor withInvalidTopic(String topicName){ - this.invalidTopic = topicName; - return this; - } public KafkaProcessor withValidateReadMessages(Function validate){ this.validateReadMessages = validate; return this; @@ -68,25 +62,19 @@ public ReadinessState process(ComponentRunner runner){ KafkaComponent kafkaComponent = runner.getComponent(kafkaComponentName, KafkaComponent.class); LinkedList outputMessages = new LinkedList<>(kafkaComponent.readMessages(readTopic)); LinkedList outputErrors = null; - LinkedList outputInvalids = null; if (errorTopic != null) { outputErrors = new LinkedList<>(kafkaComponent.readMessages(errorTopic)); } - if (invalidTopic != null) { - outputInvalids = new LinkedList<>(kafkaComponent.readMessages(invalidTopic)); - } - Boolean validated = validateReadMessages.apply(new KafkaMessageSet(outputMessages,outputErrors,outputInvalids)); + Boolean validated = validateReadMessages.apply(new KafkaMessageSet(outputMessages,outputErrors)); if(validated == null){ validated = false; } if(validated){ messages.addAll(outputMessages); errors.addAll(outputErrors); - invalids.addAll(outputInvalids); outputMessages.clear(); outputErrors.clear(); - outputInvalids.clear(); return ReadinessState.READY; } return ReadinessState.NOT_READY; @@ -94,7 +82,7 @@ public ReadinessState process(ComponentRunner runner){ @SuppressWarnings("unchecked") public ProcessorResult getResult(){ ProcessorResult.Builder builder = new ProcessorResult.Builder(); - return builder.withResult(provideResult.apply(new KafkaMessageSet(messages,errors,invalids))).withProcessErrors(errors).withProcessInvalids(invalids).build(); + return builder.withResult(provideResult.apply(new KafkaMessageSet(messages,errors))).withProcessErrors(errors).build(); } } diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index 3c4310dd41..2cf9bbfc16 100644 --- a/metron-platform/metron-parsers/README.md +++ b/metron-platform/metron-parsers/README.md @@ -30,7 +30,7 @@ Data flows through the parser bolt via kafka and into the `enrichments` topology in kafka. Errors are collected with the context of the error (e.g. stacktrace) and original message causing the error and sent to an `error` queue. Invalid messages as determined by global validation -functions are sent to an `invalid` queue. +functions are also treated as errors and sent to an `error` queue. ##Message Format @@ -277,9 +277,6 @@ usage: start_parser_topology.sh -ewp,--error_writer_p Error Writer Parallelism Hint -h,--help This screen - -iwnt,--invalid_writer_num_tasks Invalid Writer Num Tasks - -iwp,--invalid_writer_p Invalid Message Writer - Parallelism Hint -k,--kafka Kafka Broker URL -mt,--message_timeout Message Timeout in Seconds -mtp,--max_task_parallelism Max task parallelism @@ -365,9 +362,6 @@ be customized by modifying the arguments sent to this utility. * The Error Message Writer Bolt * `--error_writer_num_tasks` : The number of tasks for the error writer bolt * `--error_writer_p` : The parallelism hint for the error writer bolt -* The Invalid Message Writer Bolt - * `--invalid_writer_num_tasks` : The number of tasks for the error writer bolt - * `--invalid_writer_p` : The parallelism hint for the error writer bolt Finally, if workers and executors are new to you, the following might be of use to you: * [Understanding the Parallelism of a Storm Topology](http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/) diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 20c15ced01..0b1ea3d64f 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -18,29 +18,38 @@ package org.apache.metron.parsers.bolt; import org.apache.commons.lang3.StringUtils; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredParserBolt; +import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.FieldValidator; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.dsl.Context; import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.parsers.filters.Filters; -import org.apache.metron.common.configuration.FieldTransformer; +import org.apache.metron.common.error.MetronError; +import org.apache.metron.common.message.MessageGetStrategy; +import org.apache.metron.common.message.MessageGetters; import org.apache.metron.common.utils.ErrorUtils; +import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; +import org.apache.metron.parsers.filters.Filters; import org.apache.metron.parsers.interfaces.MessageFilter; import org.apache.metron.parsers.interfaces.MessageParser; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; public class ParserBolt extends ConfiguredParserBolt implements Serializable { @@ -51,6 +60,7 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable { private MessageFilter filter; private WriterHandler writer; private org.apache.metron.common.dsl.Context stellarContext; + private transient MessageGetStrategy messageGetStrategy; public ParserBolt( String zookeeperUrl , String sensorType , MessageParser parser @@ -72,6 +82,7 @@ public ParserBolt withMessageFilter(MessageFilter filter) { @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { super.prepare(stormConf, context, collector); + messageGetStrategy = MessageGetters.DEFAULT_BYTES_FROM_POSITION.get(); this.collector = collector; initializeStellar(); if(getSensorParserConfig() != null && filter == null) { @@ -109,7 +120,7 @@ protected void initializeStellar() { @SuppressWarnings("unchecked") @Override public void execute(Tuple tuple) { - byte[] originalMessage = tuple.getBinary(0); + byte[] originalMessage = (byte[]) messageGetStrategy.get(tuple); SensorParserConfig sensorParserConfig = getSensorParserConfig(); try { //we want to ack the tuple in the situation where we have are not doing a bulk write @@ -128,12 +139,22 @@ public void execute(Tuple tuple) { } if (parser.validate(message) && (filter == null || filter.emitTuple(message, stellarContext))) { numWritten++; - if(!isGloballyValid(message, fieldValidations)) { - message.put(Constants.SENSOR_TYPE, getSensorType()+ ".invalid"); - collector.emit(Constants.INVALID_STREAM, new Values(message)); + List failedValidators = getFailedValidators(message, fieldValidations); + if(failedValidators.size() > 0) { + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.PARSER_INVALID) + .withSensorType(getSensorType()) + .addRawMessage(message); + Set errorFields = failedValidators.stream() + .flatMap(fieldValidator -> fieldValidator.getInput().stream()) + .collect(Collectors.toSet()); + if (!errorFields.isEmpty()) { + error.withErrorFields(errorFields); + } + ErrorUtils.handleError(collector, error); } else { - writer.write(getSensorType(), tuple, message, getConfigurations()); + writer.write(getSensorType(), tuple, message, getConfigurations(), messageGetStrategy); } } } @@ -145,28 +166,28 @@ public void execute(Tuple tuple) { collector.ack(tuple); } } catch (Throwable ex) { - ErrorUtils.handleError( collector - , ex - , Constants.ERROR_STREAM - , Optional.of(getSensorType()) - , Optional.ofNullable(originalMessage) - ); + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.PARSER_ERROR) + .withThrowable(ex) + .withSensorType(getSensorType()) + .addRawMessage(originalMessage); + ErrorUtils.handleError(collector, error); collector.ack(tuple); } } - private boolean isGloballyValid(JSONObject input, List validators) { + private List getFailedValidators(JSONObject input, List validators) { + List failedValidators = new ArrayList<>(); for(FieldValidator validator : validators) { if(!validator.isValid(input, getConfigurations().getGlobalConfig(), stellarContext)) { - return false; + failedValidators.add(validator); } } - return true; + return failedValidators; } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream(Constants.INVALID_STREAM, new Fields("message")); declarer.declareStream(Constants.ERROR_STREAM, new Fields("message")); } } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java index 8eb065614d..ef7288b80e 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java @@ -18,23 +18,27 @@ package org.apache.metron.parsers.bolt; +import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ParserConfigurations; +import org.apache.metron.common.error.MetronError; +import org.apache.metron.common.message.MessageGetStrategy; +import org.apache.metron.common.message.MessageGetters; +import org.apache.metron.common.utils.ErrorUtils; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Tuple; -import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.ParserConfigurations; -import org.apache.metron.common.utils.ErrorUtils; import org.json.simple.JSONObject; import java.util.Map; -import java.util.Optional; public class WriterBolt extends BaseRichBolt { private WriterHandler handler; private ParserConfigurations configuration; private String sensorType; + private Constants.ErrorType errorType = Constants.ErrorType.DEFAULT_ERROR; + private transient MessageGetStrategy messageGetStrategy; private transient OutputCollector collector; public WriterBolt(WriterHandler handler, ParserConfigurations configuration, String sensorType) { this.handler = handler; @@ -42,9 +46,15 @@ public WriterBolt(WriterHandler handler, ParserConfigurations configuration, Str this.sensorType = sensorType; } + public WriterBolt withErrorType(Constants.ErrorType errorType) { + this.errorType = errorType; + return this; + } + @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.collector = collector; + messageGetStrategy = MessageGetters.DEFAULT_JSON_FROM_FIELD.get(); handler.init(stormConf, collector, configuration); } @@ -65,18 +75,18 @@ private JSONObject getMessage(Tuple tuple) { public void execute(Tuple tuple) { JSONObject message = null; try { - message = (JSONObject)((JSONObject) tuple.getValueByField("message")).clone(); - handler.write(sensorType, tuple, message, configuration); + message = (JSONObject) messageGetStrategy.get(tuple); + handler.write(sensorType, tuple, message, configuration, messageGetStrategy); if(!handler.handleAck()) { collector.ack(tuple); } } catch (Throwable e) { - ErrorUtils.handleError( collector - , e - , Constants.ERROR_STREAM - , Optional.of(sensorType) - , Optional.ofNullable(message) - ); + MetronError error = new MetronError() + .withErrorType(errorType) + .withThrowable(e) + .withSensorType(sensorType) + .addRawMessage(message); + ErrorUtils.handleError(collector, error); collector.ack(tuple); } } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java index 38425b5891..3273ca7971 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java @@ -18,6 +18,7 @@ package org.apache.metron.parsers.bolt; +import org.apache.metron.common.message.MessageGetStrategy; import org.apache.storm.task.OutputCollector; import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.ParserConfigurations; @@ -80,11 +81,12 @@ public void write( String sensorType , Tuple tuple , JSONObject message , ParserConfigurations configurations + , MessageGetStrategy messageGetStrategy ) throws Exception { - writerComponent.write(sensorType, tuple, message, messageWriter, writerTransformer.apply(configurations)); + writerComponent.write(sensorType, tuple, message, messageWriter, writerTransformer.apply(configurations), messageGetStrategy); } - public void errorAll(String sensorType, Throwable e) { - writerComponent.errorAll(sensorType, e); + public void errorAll(String sensorType, Throwable e, MessageGetStrategy messageGetStrategy) { + writerComponent.errorAll(sensorType, e, messageGetStrategy); } } diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java index 5f1927e9bc..aeac33caa2 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java @@ -57,8 +57,6 @@ public class ParserTopologyBuilder { * @param spoutNumTasks Number of tasks for the spout * @param parserParallelism Parallelism hint for the parser bolt * @param parserNumTasks Number of tasks for the parser bolt - * @param invalidWriterParallelism Parallelism hint for the bolt that handles invalid data - * @param invalidWriterNumTasks Number of tasks for the bolt that handles invalid data * @param errorWriterParallelism Parallelism hint for the bolt that handles errors * @param errorWriterNumTasks Number of tasks for the bolt that handles errors * @param kafkaSpoutConfig Configuration options for the kafka spout @@ -73,8 +71,6 @@ public static TopologyBuilder build(String zookeeperUrl, int spoutNumTasks, int parserParallelism, int parserNumTasks, - int invalidWriterParallelism, - int invalidWriterNumTasks, int errorWriterParallelism, int errorWriterNumTasks, EnumMap kafkaSpoutConfig @@ -104,14 +100,6 @@ public static TopologyBuilder build(String zookeeperUrl, .shuffleGrouping("parserBolt", Constants.ERROR_STREAM); } - // create the invalid bolt, if needed - if (invalidWriterNumTasks > 0) { - WriterBolt invalidBolt = createInvalidBolt(brokerUrl, sensorType, configs, parserConfig); - builder.setBolt("invalidMessageWriter", invalidBolt, invalidWriterParallelism) - .setNumTasks(invalidWriterNumTasks) - .shuffleGrouping("parserBolt", Constants.INVALID_STREAM); - } - return builder; } @@ -161,29 +149,6 @@ private static ParserBolt createParserBolt(String zookeeperUrl, String brokerUrl return new ParserBolt(zookeeperUrl, sensorType, parser, writerHandler); } - /** - * Create a bolt that handles invalid messages. - * - * @param brokerUrl The Kafka Broker URL - * @param sensorType Type of sensor that is being consumed. - * @param configs - * @param parserConfig - * @return A Storm bolt that handles invalid messages. - */ - private static WriterBolt createInvalidBolt(String brokerUrl, String sensorType, ParserConfigurations configs, SensorParserConfig parserConfig) { - - // create writer - if not configured uses a sensible default - AbstractWriter writer = parserConfig.getErrorWriterClassName() == null - ? new KafkaWriter(brokerUrl).withTopic(Constants.DEFAULT_PARSER_INVALID_TOPIC).withConfigPrefix("invalid") - : ReflectionUtils.createInstance(parserConfig.getWriterClassName()); - writer.configure(sensorType, new ParserWriterConfiguration(configs)); - - // create a writer handler - WriterHandler writerHandler = createWriterHandler(writer); - - return new WriterBolt(writerHandler, configs, sensorType); - } - /** * Create a bolt that handles error messages. * @@ -197,14 +162,14 @@ private static WriterBolt createErrorBolt(String brokerUrl, String sensorType, P // create writer - if not configured uses a sensible default AbstractWriter writer = parserConfig.getErrorWriterClassName() == null - ? new KafkaWriter(brokerUrl).withTopic(Constants.DEFAULT_PARSER_ERROR_TOPIC).withConfigPrefix("error") + ? new KafkaWriter(brokerUrl).withTopic((String) configs.getGlobalConfig().get("parser.error.topic")).withConfigPrefix("error") : ReflectionUtils.createInstance(parserConfig.getWriterClassName()); writer.configure(sensorType, new ParserWriterConfiguration(configs)); // create a writer handler WriterHandler writerHandler = createWriterHandler(writer); - return new WriterBolt(writerHandler, configs, sensorType); + return new WriterBolt(writerHandler, configs, sensorType).withErrorType(Constants.ErrorType.PARSER_ERROR); } /** diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java index 5ea561c709..2bf484ec4e 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java @@ -302,8 +302,6 @@ public static void main(String[] args) { spoutNumTasks, parserParallelism, parserNumTasks, - invalidParallelism, - invalidNumTasks, errorParallelism, errorNumTasks, spoutConfig diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index 6b00998e6a..b3e15b2c9d 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -17,8 +17,11 @@ */ package org.apache.metron.parsers.bolt; +import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.*; +import org.apache.metron.common.error.MetronError; +import org.apache.metron.test.error.MetronErrorJSONMatcher; import org.apache.metron.test.utils.UnitTestHelper; import org.apache.storm.task.OutputCollector; import org.apache.storm.tuple.Tuple; @@ -151,6 +154,64 @@ public Map getParserConfig() { verify(parser, times(0)).validate(any()); verify(writer, times(0)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), any()); verify(outputCollector, times(1)).ack(tuple); + + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.PARSER_ERROR) + .withThrowable(new NullPointerException()) + .withSensorType(sensorType) + .addRawMessage(sampleBinary); + verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); + } + + @Test + public void testInvalid() throws Exception { + String sensorType = "yaf"; + ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) { + @Override + protected ParserConfigurations defaultConfigurations() { + return new ParserConfigurations() { + @Override + public SensorParserConfig getSensorParserConfig(String sensorType) { + return new SensorParserConfig() { + @Override + public Map getParserConfig() { + return new HashMap() {{ + }}; + } + + + }; + } + }; + } + + }; + + buildGlobalConfig(parserBolt); + + parserBolt.setCuratorFramework(client); + parserBolt.setTreeCache(cache); + parserBolt.prepare(new HashMap(), topologyContext, outputCollector); + byte[] sampleBinary = "some binary message".getBytes(); + + when(tuple.getBinary(0)).thenReturn(sampleBinary); + JSONObject parsedMessage = new JSONObject(); + parsedMessage.put("field", "invalidValue"); + List messageList = new ArrayList<>(); + messageList.add(parsedMessage); + when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messageList)); + when(parser.validate(parsedMessage)).thenReturn(true); + parserBolt.execute(tuple); + + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.PARSER_INVALID) + .withSensorType(sensorType) + .withErrorFields(new HashSet() {{ add("field"); }}) + .addRawMessage(new JSONObject(){{ + put("field", "invalidValue"); + put("source.type", "yaf"); + }}); + verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); } @Test @@ -531,6 +592,16 @@ public Map getParserConfig() { } + protected void buildGlobalConfig(ParserBolt parserBolt) { + HashMap globalConfig = new HashMap<>(); + Map fieldValidation = new HashMap<>(); + fieldValidation.put("input", Arrays.asList("field")); + fieldValidation.put("validation", "STELLAR"); + fieldValidation.put("config", new HashMap(){{ put("condition", "field != 'invalidValue'"); }}); + globalConfig.put("fieldValidations", Arrays.asList(fieldValidation)); + parserBolt.getConfigurations().updateGlobalConfig(globalConfig); + } + private static void writeNonBatch(OutputCollector collector, ParserBolt bolt, Tuple t) { bolt.execute(t); } diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java index 4693829b1c..4511b5598e 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java @@ -17,22 +17,23 @@ */ package org.apache.metron.parsers.bolt; + import org.apache.log4j.Level; +import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.IndexingConfigurations; -import org.apache.metron.test.utils.UnitTestHelper; -import org.apache.metron.writer.BulkWriterComponent; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Tuple; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; -import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; +import org.apache.metron.common.error.MetronError; import org.apache.metron.common.writer.BulkMessageWriter; -import org.apache.metron.common.writer.MessageWriter; import org.apache.metron.common.writer.BulkWriterResponse; +import org.apache.metron.common.writer.MessageWriter; import org.apache.metron.test.bolt.BaseBoltTest; +import org.apache.metron.test.error.MetronErrorJSONMatcher; +import org.apache.metron.test.utils.UnitTestHelper; +import org.apache.metron.writer.BulkWriterComponent; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; import org.junit.Test; import org.mockito.Mock; @@ -44,7 +45,12 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.argThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class WriterBoltTest extends BaseBoltTest{ @Mock @@ -143,16 +149,24 @@ public void testNonBatchErrorPathErrorInWrite() throws Exception { ParserConfigurations configurations = getConfigurations(1); String sensorType = "test"; Tuple t = mock(Tuple.class); + when(t.toString()).thenReturn("tuple"); when(t.getValueByField(eq("message"))).thenReturn(new JSONObject()); WriterBolt bolt = new WriterBolt(new WriterHandler(writer), configurations, sensorType); bolt.prepare(new HashMap(), topologyContext, outputCollector); - doThrow(new Exception()).when(writer).write(any(), any(), any(), any()); + doThrow(new Exception("write error")).when(writer).write(any(), any(), any(), any()); verify(writer, times(1)).init(); bolt.execute(t); verify(outputCollector, times(1)).ack(t); verify(writer, times(1)).write(eq(sensorType), any(), any(), any()); verify(outputCollector, times(1)).reportError(any()); verify(outputCollector, times(0)).fail(any()); + + MetronError error = new MetronError() + .withErrorType(Constants.ErrorType.DEFAULT_ERROR) + .withThrowable(new IllegalStateException("Unhandled bulk errors in response: {java.lang.Exception: write error=[tuple]}")) + .withSensorType(sensorType) + .addRawMessage(new JSONObject()); + verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); } @Test diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java index 4ba1c43a76..a170a2c06f 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java @@ -37,6 +37,7 @@ import java.util.*; public abstract class ParserIntegrationTest extends BaseIntegrationTest { + protected static final String ERROR_TOPIC = "parser_error"; protected List inputMessages; @Test public void test() throws Exception { @@ -47,8 +48,7 @@ public void test() throws Exception { final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList() {{ add(new KafkaComponent.Topic(sensorType, 1)); add(new KafkaComponent.Topic(Constants.ENRICHMENT_TOPIC, 1)); - add(new KafkaComponent.Topic(Constants.INVALID_STREAM,1)); - add(new KafkaComponent.Topic(Constants.ERROR_STREAM,1)); + add(new KafkaComponent.Topic(ERROR_TOPIC,1)); }}); topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList()); @@ -115,13 +115,12 @@ private KafkaProcessor> getProcessor(){ return new KafkaProcessor<>() .withKafkaComponentName("kafka") .withReadTopic(Constants.ENRICHMENT_TOPIC) - .withErrorTopic(Constants.ERROR_STREAM) - .withInvalidTopic(Constants.INVALID_STREAM) + .withErrorTopic(ERROR_TOPIC) .withValidateReadMessages(new Function() { @Nullable @Override public Boolean apply(@Nullable KafkaMessageSet messageSet) { - return (messageSet.getMessages().size() + messageSet.getErrors().size() + messageSet.getInvalids().size()) == inputMessages.size(); + return (messageSet.getMessages().size() + messageSet.getErrors().size() == inputMessages.size()); } }) .withProvideResult(new Function>(){ diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java index 80b6ebdb87..73d38274e1 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java @@ -34,6 +34,7 @@ public class ParserTopologyComponent implements InMemoryComponent { private Properties topologyProperties; private String brokerUrl; private String sensorType; + private SpoutConfig.Offset offset = SpoutConfig.Offset.BEGINNING; private LocalCluster stormCluster; public static class Builder { @@ -64,15 +65,17 @@ public ParserTopologyComponent(Properties topologyProperties, String brokerUrl, this.sensorType = sensorType; } + public void setOffset(SpoutConfig.Offset offset) { + this.offset = offset; + } + @Override public void start() throws UnableToStartException { try { TopologyBuilder topologyBuilder = ParserTopologyBuilder.build(topologyProperties.getProperty("kafka.zk") , brokerUrl , sensorType - , SpoutConfig.Offset.BEGINNING - , 1 - , 1 + , offset , 1 , 1 , 1 diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java index f37b1fc642..7476bcf250 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java @@ -15,8 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.metron.writers.integration; + import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; import com.google.common.collect.Iterables; @@ -37,6 +37,8 @@ import org.apache.metron.parsers.integration.components.ParserTopologyComponent; import org.apache.metron.test.utils.UnitTestHelper; import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; import org.junit.Assert; import org.junit.Test; @@ -45,6 +47,8 @@ import java.util.*; public class WriterBoltIntegrationTest extends BaseIntegrationTest { + private static final String ERROR_TOPIC = "parser_error"; + public static class MockValidator implements FieldValidation{ @Override @@ -65,7 +69,8 @@ public void initialize(Map validationConfig, Map { "validation" : "org.apache.metron.writers.integration.WriterBoltIntegrationTest$MockValidator" } - ] + ], + "parser.error.topic":"parser_error" } */ @Multiline @@ -88,7 +93,7 @@ public void initialize(Map validationConfig, Map public static String parserConfig; @Test - public void test() throws UnableToStartException, IOException { + public void test() throws UnableToStartException, IOException, ParseException { UnitTestHelper.setLog4jLevel(CSVParser.class, org.apache.log4j.Level.FATAL); final String sensorType = "dummy"; final List inputMessages = new ArrayList() {{ @@ -100,8 +105,7 @@ public void test() throws UnableToStartException, IOException { final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties); final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList() {{ add(new KafkaComponent.Topic(sensorType, 1)); - add(new KafkaComponent.Topic(Constants.DEFAULT_PARSER_ERROR_TOPIC, 1)); - add(new KafkaComponent.Topic(Constants.DEFAULT_PARSER_INVALID_TOPIC, 1)); + add(new KafkaComponent.Topic(ERROR_TOPIC, 1)); add(new KafkaComponent.Topic(Constants.ENRICHMENT_TOPIC, 1)); }}); topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList()); @@ -131,17 +135,20 @@ public void test() throws UnableToStartException, IOException { kafkaComponent.writeMessages(sensorType, inputMessages); ProcessorResult>> result = runner.process(getProcessor()); Map> outputMessages = result.getResult(); - Assert.assertEquals(3, outputMessages.size()); + Assert.assertEquals(2, outputMessages.size()); Assert.assertEquals(1, outputMessages.get(Constants.ENRICHMENT_TOPIC).size()); Assert.assertEquals("valid", outputMessages.get(Constants.ENRICHMENT_TOPIC).get(0).get("action")); - Assert.assertEquals(1, outputMessages.get(Constants.DEFAULT_PARSER_ERROR_TOPIC).size()); - Assert.assertEquals("error", outputMessages.get(Constants.DEFAULT_PARSER_ERROR_TOPIC).get(0).get("rawMessage")); - Assert.assertTrue(Arrays.equals(listToBytes(outputMessages.get(Constants.DEFAULT_PARSER_ERROR_TOPIC).get(0).get("rawMessage_bytes")) - , "error".getBytes() - ) - ); - Assert.assertEquals(1, outputMessages.get(Constants.DEFAULT_PARSER_INVALID_TOPIC).size()); - Assert.assertEquals("invalid", outputMessages.get(Constants.DEFAULT_PARSER_INVALID_TOPIC).get(0).get("action")); + Assert.assertEquals(2, outputMessages.get(ERROR_TOPIC).size()); + JSONObject invalidMessage = outputMessages.get(ERROR_TOPIC).get(0); + Assert.assertEquals(Constants.ErrorType.PARSER_INVALID.getType(), invalidMessage.get(Constants.ErrorFields.ERROR_TYPE.getName())); + JSONObject rawMessage = JSONUtils.INSTANCE.load((String) invalidMessage.get(Constants.ErrorFields.RAW_MESSAGE.getName()), JSONObject.class); + Assert.assertEquals("foo", rawMessage.get("dummy")); + Assert.assertEquals("invalid", rawMessage.get("action")); + JSONObject errorMessage = outputMessages.get(ERROR_TOPIC).get(1); + Assert.assertEquals(Constants.ErrorType.PARSER_ERROR.getType(), errorMessage.get(Constants.ErrorFields.ERROR_TYPE.getName())); + Assert.assertEquals("error", errorMessage.get(Constants.ErrorFields.RAW_MESSAGE.getName())); + // It's unclear if we need a rawMessageBytes field so commenting out for now + //Assert.assertTrue(Arrays.equals(listToBytes(errorMessage.get(Constants.ErrorFields.RAW_MESSAGE_BYTES.getName())), "error".getBytes())); } finally { if(runner != null) { @@ -182,13 +189,12 @@ private KafkaProcessor>> getProcessor(){ return new KafkaProcessor<>() .withKafkaComponentName("kafka") .withReadTopic(Constants.ENRICHMENT_TOPIC) - .withErrorTopic(Constants.DEFAULT_PARSER_ERROR_TOPIC) - .withInvalidTopic(Constants.DEFAULT_PARSER_INVALID_TOPIC) + .withErrorTopic(ERROR_TOPIC) .withValidateReadMessages(new Function() { @Nullable @Override public Boolean apply(@Nullable KafkaMessageSet messageSet) { - return (messageSet.getMessages().size() == 1) && (messageSet.getErrors().size() == 1) && (messageSet.getInvalids().size() ==1); + return (messageSet.getMessages().size() == 1) && (messageSet.getErrors().size() == 2); } }) .withProvideResult(new Function>>(){ @@ -197,8 +203,7 @@ public Boolean apply(@Nullable KafkaMessageSet messageSet) { public Map> apply(@Nullable KafkaMessageSet messageSet) { return new HashMap>() {{ put(Constants.ENRICHMENT_TOPIC, loadMessages(messageSet.getMessages())); - put(Constants.DEFAULT_PARSER_ERROR_TOPIC, loadMessages(messageSet.getErrors())); - put(Constants.DEFAULT_PARSER_INVALID_TOPIC, loadMessages(messageSet.getInvalids())); + put(ERROR_TOPIC, loadMessages(messageSet.getErrors())); }}; } }); diff --git a/metron-platform/metron-solr/src/main/config/solr.properties b/metron-platform/metron-solr/src/main/config/solr.properties index 8489d8e2b9..d8dd25f1bc 100644 --- a/metron-platform/metron-solr/src/main/config/solr.properties +++ b/metron-platform/metron-solr/src/main/config/solr.properties @@ -26,7 +26,7 @@ kafka.start=WHERE_I_LEFT_OFF ##### Indexing ##### index.input.topic=indexing -index.error.topic=indexing_error +index.error.topic=indexing writer.class.name=org.apache.metron.solr.writer.SolrWriter ##### Metrics ##### diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java index abf3a8a279..c209ef346a 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java @@ -18,14 +18,17 @@ package org.apache.metron.solr.integration; import com.google.common.base.Function; -import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.Configurations; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.interfaces.FieldNameConverter; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.enrichment.integration.utils.SampleUtil; import org.apache.metron.indexing.integration.IndexingIntegrationTest; -import org.apache.metron.integration.*; +import org.apache.metron.integration.ComponentRunner; +import org.apache.metron.integration.InMemoryComponent; +import org.apache.metron.integration.Processor; +import org.apache.metron.integration.ProcessorResult; +import org.apache.metron.integration.ReadinessState; import org.apache.metron.integration.components.KafkaComponent; import org.apache.metron.integration.components.ZKServerComponent; import org.apache.metron.solr.integration.components.SolrComponent; @@ -89,7 +92,7 @@ public ReadinessState process(ComponentRunner runner) { throw new IllegalStateException("Unable to retrieve indexed documents.", e); } if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) { - errors = kafkaComponent.readMessages(Constants.INDEXING_ERROR_TOPIC); + errors = kafkaComponent.readMessages(ERROR_TOPIC); if(errors.size() > 0){ return ReadinessState.READY; } diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/error/MetronErrorJSONMatcher.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/error/MetronErrorJSONMatcher.java new file mode 100644 index 0000000000..ad2428372f --- /dev/null +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/error/MetronErrorJSONMatcher.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.test.error; + +import org.apache.storm.tuple.Values; +import org.json.simple.JSONObject; +import org.mockito.ArgumentMatcher; + +public class MetronErrorJSONMatcher extends ArgumentMatcher { + + private JSONObject expected; + + public MetronErrorJSONMatcher(JSONObject expected) { + this.expected = expected; + } + + @Override + public boolean matches(Object o) { + Values values = (Values) o; + JSONObject actual = (JSONObject) values.get(0); + actual.remove("timestamp"); + expected.remove("timestamp"); + actual.remove("stack"); + expected.remove("stack"); + return actual.equals(expected); + } +} diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java index 124ffd3b36..0a9e5144b7 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java @@ -18,18 +18,26 @@ package org.apache.metron.writer; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.tuple.Tuple; import com.google.common.collect.Iterables; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.writer.WriterConfiguration; +import org.apache.metron.common.error.MetronError; +import org.apache.metron.common.message.MessageGetStrategy; +import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.common.writer.BulkMessageWriter; import org.apache.metron.common.writer.BulkWriterResponse; -import org.apache.metron.common.utils.ErrorUtils; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; public class BulkWriterComponent { public static final Logger LOG = LoggerFactory @@ -60,18 +68,23 @@ public void commit(BulkWriterResponse response) { commit(response.getSuccesses()); } - public void error(Throwable e, Iterable tuples) { + public void error(String sensorType, Throwable e, Iterable tuples, MessageGetStrategy messageGetStrategy) { tuples.forEach(t -> collector.ack(t)); + MetronError error = new MetronError() + .withSensorType(sensorType) + .withErrorType(Constants.ErrorType.INDEXING_ERROR) + .withThrowable(e); if(!Iterables.isEmpty(tuples)) { LOG.error("Failing " + Iterables.size(tuples) + " tuples", e); - ErrorUtils.handleError(collector, e, Constants.ERROR_STREAM); } + tuples.forEach(t -> error.addRawMessage(messageGetStrategy.get(t))); + ErrorUtils.handleError(collector, error); } - public void error(BulkWriterResponse errors) { + public void error(String sensorType, BulkWriterResponse errors, MessageGetStrategy messageGetStrategy) { Map> errorMap = errors.getErrors(); for(Map.Entry> entry : errorMap.entrySet()) { - error(entry.getKey(), entry.getValue()); + error(sensorType, entry.getKey(), entry.getValue(), messageGetStrategy); } } @@ -80,24 +93,25 @@ protected Collection createTupleCollection() { } - public void errorAll(Throwable e) { - for(Map.Entry> kv : sensorTupleMap.entrySet()) { - error(e, kv.getValue()); - sensorTupleMap.remove(kv.getKey()); - sensorMessageMap.remove(kv.getKey()); + public void errorAll(Throwable e, MessageGetStrategy messageGetStrategy) { + for(String key : new HashSet<>(sensorTupleMap.keySet())) { + errorAll(key, e, messageGetStrategy); } } - public void errorAll(String sensorType, Throwable e) { - error(e, Optional.ofNullable(sensorTupleMap.get(sensorType)).orElse(new ArrayList<>())); + public void errorAll(String sensorType, Throwable e, MessageGetStrategy messageGetStrategy) { + Collection tuples = Optional.ofNullable(sensorTupleMap.get(sensorType)).orElse(new ArrayList<>()); + error(sensorType, e, tuples, messageGetStrategy); sensorTupleMap.remove(sensorType); sensorMessageMap.remove(sensorType); } + public void write( String sensorType , Tuple tuple , MESSAGE_T message , BulkMessageWriter bulkMessageWriter , WriterConfiguration configurations + , MessageGetStrategy messageGetStrategy ) throws Exception { if(!configurations.isEnabled(sensorType)) { @@ -129,13 +143,13 @@ public void write( String sensorType } if(handleError) { - error(response); + error(sensorType, response, messageGetStrategy); } else if (response.hasErrors()) { throw new IllegalStateException("Unhandled bulk errors in response: " + response.getErrors()); } } catch (Throwable e) { if(handleError) { - error(e, tupleList); + error(sensorType, e, tupleList, messageGetStrategy); } else { throw e; diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java index 66c4c73d43..085ca5c182 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java @@ -17,27 +17,27 @@ */ package org.apache.metron.writer.bolt; -import org.apache.metron.common.bolt.ConfiguredIndexingBolt; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; import org.apache.metron.common.Constants; +import org.apache.metron.common.bolt.ConfiguredIndexingBolt; import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration; import org.apache.metron.common.configuration.writer.WriterConfiguration; -import org.apache.metron.common.writer.MessageWriter; +import org.apache.metron.common.message.MessageGetStrategy; +import org.apache.metron.common.message.MessageGetters; import org.apache.metron.common.utils.MessageUtils; import org.apache.metron.common.writer.BulkMessageWriter; +import org.apache.metron.common.writer.MessageWriter; import org.apache.metron.writer.BulkWriterComponent; import org.apache.metron.writer.WriterToBulkWriter; -import org.apache.metron.writer.message.MessageGetter; -import org.apache.metron.writer.message.MessageGetters; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.Map; import java.util.function.Function; public class BulkMessageWriterBolt extends ConfiguredIndexingBolt { @@ -46,8 +46,9 @@ public class BulkMessageWriterBolt extends ConfiguredIndexingBolt { .getLogger(BulkMessageWriterBolt.class); private BulkMessageWriter bulkMessageWriter; private BulkWriterComponent writerComponent; - private String messageGetterStr = MessageGetters.NAMED.name(); - private transient MessageGetter messageGetter = null; + private String messageGetStrategyType = MessageGetters.DEFAULT_JSON_FROM_FIELD.name(); + private String messageGetField; + private transient MessageGetStrategy messageGetStrategy; private transient OutputCollector collector; private transient Function configurationTransformation; public BulkMessageWriterBolt(String zookeeperUrl) { @@ -64,8 +65,13 @@ public BulkMessageWriterBolt withMessageWriter(MessageWriter message return this; } - public BulkMessageWriterBolt withMessageGetter(String messageGetter) { - this.messageGetterStr = messageGetter; + public BulkMessageWriterBolt withMessageGetter(String messageGetStrategyType) { + this.messageGetStrategyType = messageGetStrategyType; + return this; + } + + public BulkMessageWriterBolt withMessageGetterField(String messageGetField) { + this.messageGetField = messageGetField; return this; } @@ -74,7 +80,11 @@ public void prepare(Map stormConf, TopologyContext context, OutputCollector coll this.writerComponent = new BulkWriterComponent<>(collector); this.collector = collector; super.prepare(stormConf, context, collector); - messageGetter = MessageGetters.valueOf(messageGetterStr); + if (messageGetField != null) { + messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get(messageGetField); + } else { + messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get(); + } if(bulkMessageWriter instanceof WriterToBulkWriter) { configurationTransformation = WriterToBulkWriter.TRANSFORMATION; } @@ -93,7 +103,7 @@ public void prepare(Map stormConf, TopologyContext context, OutputCollector coll @SuppressWarnings("unchecked") @Override public void execute(Tuple tuple) { - JSONObject message = messageGetter.getMessage(tuple); + JSONObject message = (JSONObject) messageGetStrategy.get(tuple); String sensorType = MessageUtils.getSensorType(message); try { @@ -107,6 +117,7 @@ public void execute(Tuple tuple) { , message , bulkMessageWriter , writerConfiguration + , messageGetStrategy ); LOG.trace("Writing enrichment message: {}", message); } diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java new file mode 100644 index 0000000000..c560b30686 --- /dev/null +++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.writer; + +import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.writer.WriterConfiguration; +import org.apache.metron.common.error.MetronError; +import org.apache.metron.common.message.MessageGetStrategy; +import org.apache.metron.common.message.MessageGetters; +import org.apache.metron.common.utils.ErrorUtils; +import org.apache.metron.common.writer.BulkMessageWriter; +import org.apache.metron.common.writer.BulkWriterResponse; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; +import org.json.simple.JSONObject; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.verifyStatic; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({BulkWriterComponent.class, ErrorUtils.class}) +public class BulkWriterComponentTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Mock + private OutputCollector collector; + + @Mock + private BulkMessageWriter bulkMessageWriter; + + @Mock + private WriterConfiguration configurations; + + @Mock + private Tuple tuple1; + + @Mock + private Tuple tuple2; + + @Mock + private MessageGetStrategy messageGetStrategy; + + private String sensorType = "testSensor"; + private List tupleList; + private JSONObject message1 = new JSONObject(); + private JSONObject message2 = new JSONObject(); + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + mockStatic(ErrorUtils.class); + message1.put("value", "message1"); + message2.put("value", "message2"); + when(tuple1.getValueByField("message")).thenReturn(message1); + when(tuple2.getValueByField("message")).thenReturn(message2); + tupleList = Arrays.asList(tuple1, tuple2); + when(configurations.isEnabled(any())).thenReturn(true); + when(configurations.getBatchSize(any())).thenReturn(2); + when(messageGetStrategy.get(tuple1)).thenReturn(message1); + when(messageGetStrategy.get(tuple2)).thenReturn(message2); + } + + @Test + public void writeShouldProperlyAckTuplesInBatch() throws Exception { + BulkWriterResponse response = new BulkWriterResponse(); + response.addAllSuccesses(tupleList); + + when(bulkMessageWriter.write(sensorType, configurations, Arrays.asList(tuple1, tuple2), Arrays.asList(message1, message2))).thenReturn(response); + + BulkWriterComponent bulkWriterComponent = new BulkWriterComponent<>(collector); + bulkWriterComponent.write(sensorType, tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy); + + verify(bulkMessageWriter, times(0)).write(sensorType, configurations, Collections.singletonList(tuple1), Collections.singletonList(message1)); + verify(collector, times(0)).ack(tuple1); + verify(collector, times(0)).ack(tuple2); + + bulkWriterComponent.write(sensorType, tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy); + + verify(collector, times(1)).ack(tuple1); + verify(collector, times(1)).ack(tuple2); + verifyStatic(times(0)); + ErrorUtils.handleError(eq(collector), any(MetronError.class)); + } + + @Test + public void writeShouldProperlyHandleWriterErrors() throws Exception { + Throwable e = new Exception("test exception"); + MetronError error = new MetronError() + .withSensorType(sensorType) + .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Arrays.asList(message1, message2)); + BulkWriterResponse response = new BulkWriterResponse(); + response.addAllErrors(e, tupleList); + + when(bulkMessageWriter.write(sensorType, configurations, Arrays.asList(tuple1, tuple2), Arrays.asList(message1, message2))).thenReturn(response); + + BulkWriterComponent bulkWriterComponent = new BulkWriterComponent<>(collector); + bulkWriterComponent.write(sensorType, tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy); + bulkWriterComponent.write(sensorType, tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy); + + verifyStatic(times(1)); + ErrorUtils.handleError(collector, error); + } + + @Test + public void writeShouldThrowExceptionWhenHandleErrorIsFalse() throws Exception { + exception.expect(IllegalStateException.class); + + Throwable e = new Exception("test exception"); + BulkWriterResponse response = new BulkWriterResponse(); + response.addAllErrors(e, tupleList); + + when(bulkMessageWriter.write(sensorType, configurations, Arrays.asList(tuple1, tuple2), Arrays.asList(message1, message2))).thenReturn(response); + + BulkWriterComponent bulkWriterComponent = new BulkWriterComponent<>(collector, true, false); + bulkWriterComponent.write(sensorType, tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy); + bulkWriterComponent.write(sensorType, tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy); + } + + @Test + public void writeShouldProperlyHandleWriterException() throws Exception { + Throwable e = new Exception("test exception"); + MetronError error = new MetronError() + .withSensorType(sensorType) + .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Arrays.asList(message1, message2)); + BulkWriterResponse response = new BulkWriterResponse(); + response.addAllErrors(e, tupleList); + + when(bulkMessageWriter.write(sensorType, configurations, Arrays.asList(tuple1, tuple2), Arrays.asList(message1, message2))).thenThrow(e); + + BulkWriterComponent bulkWriterComponent = new BulkWriterComponent<>(collector); + bulkWriterComponent.write(sensorType, tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy); + bulkWriterComponent.write(sensorType, tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy); + + verifyStatic(times(1)); + ErrorUtils.handleError(collector, error); + } + + @Test + public void errorAllShouldClearMapsAndHandleErrors() throws Exception { + Throwable e = new Exception("test exception"); + MetronError error1 = new MetronError() + .withSensorType("sensor1") + .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message1)); + MetronError error2 = new MetronError() + .withSensorType("sensor2") + .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message2)); + + BulkWriterComponent bulkWriterComponent = new BulkWriterComponent<>(collector); + bulkWriterComponent.write("sensor1", tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy); + bulkWriterComponent.write("sensor2", tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy); + bulkWriterComponent.errorAll(e, messageGetStrategy); + + verifyStatic(times(1)); + ErrorUtils.handleError(collector, error1); + ErrorUtils.handleError(collector, error2); + + bulkWriterComponent.write("sensor1", tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy); + verify(bulkMessageWriter, times(0)).write(sensorType, configurations, Collections.singletonList(tuple1), Collections.singletonList(message1)); + } + + +}