diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template index 7521af76f5..18c5d9b58d 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template @@ -122,76 +122,166 @@ } ], "properties": { + /* + * WARNING + * + * Because Metron inserts many distinct bro records into a single ElasticSearch index + * by default, it encounters field collisions due to field name reuse across bro logs. + * + * Be careful when modifying this file to not unintentionally affect other logs. + * For instance, the "version" field exists in the HTTP, SSL, and SSH logs. If you + * were to only consider the SSH log, you would set the type to integer, but because + * in the SSL and HTTP logs version is a string, we must set the type to string. + */ + /* + * Metron-specific fields + */ + "source:type": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Widely-used Bro fields (potentially renamed during Metron ingest) + */ "timestamp": { "type": "date", "format": "epoch_millis" }, - "source:type": { + "uid": { "type": "string", "index": "not_analyzed" }, - "ip_dst_addr": { + "ip_src_addr": { "type": "ip" }, - "ip_dst_port": { + "ip_src_port": { "type": "integer" }, - "ip_src_addr": { + "ip_dst_addr": { "type": "ip" }, - "ip_src_port": { + "ip_dst_port": { "type": "integer" }, - "status_code": { + /* + * HTTP log support + * https://www.bro.org/sphinx/scripts/base/protocols/http/main.bro.html#type-HTTP::Info + * + * Notable Fields + * Field: password + * Notes: Field exists in the HTTP and FTP logs + * + * Field: capture_password + * Notes: Field exists in the HTTP and FTP logs + * + * Field: trans_depth + * Notes: Field exists in the HTTP and SMTP logs + * + * Field: user_agent + * Notes: Field exists in the HTTP and SMTP logs + * + * Field: version + * Notes: Field exists in the HTTP, SSL, and SSH logs + * + * Field: host + * Notes: Field exists in the HTTP and Software logs + * + * Field: username + * Notes: Field exists in the HTTP and RADIUS logs + */ + "trans_depth": { "type": "integer" }, "method": { "type": "string", "index": "not_analyzed" }, - "protocol": { + "host": { "type": "string", "index": "not_analyzed" }, - "request_body_len": { - "type": "long" - }, "uri": { "type": "string", "index": "not_analyzed", "ignore_above": 8191 }, - "uid": { + "referrer": { "type": "string", "index": "not_analyzed" }, - "referrer": { + "version": { "type": "string", "index": "not_analyzed" }, - "trans_depth": { + "user_agent": { + "type": "string" + }, + "request_body_len": { + "type": "long" + }, + "response_body_len": { + "type": "long" + }, + "status_code": { "type": "integer" }, - "host": { + "status_msg": { "type": "string", "index": "not_analyzed" }, - "status_msg": { + "username": { "type": "string", "index": "not_analyzed" }, - "response_body_len": { - "type": "long" + "password": { + "type": "string", + "index": "not_analyzed" }, - "user_agent": { - "type": "string" + "capture_password": { + "type": "boolean" + }, + /* + * DNS log support + * https://www.bro.org/sphinx/scripts/base/protocols/dns/main.bro.html#type-DNS::Info + * + * Notable Fields + * Field: proto + * Notes: Field exists in the DNS, Conn, DPD, and Notice logs + * + * Field: trans_id + * Notes: Field exists in the DNS and DHCP logs + */ + "proto": { + "type": "string", + "index": "not_analyzed" + }, + "trans_id": { + "type": "long" }, "query": { "type": "string", "index": "not_analyzed" }, - "answers": { - "type": "string" + "qclass": { + "type": "integer" + }, + "qclass_name": { + "type": "string", + "index": "not_analyzed" + }, + "qtype": { + "type": "integer" + }, + "qtype_name": { + "type": "string", + "index": "not_analyzed" + }, + "rcode": { + "type": "integer" + }, + "rcode_name": { + "type": "string", + "index": "not_analyzed" }, "AA": { "type": "boolean" @@ -199,43 +289,680 @@ "TC": { "type": "boolean" }, - "RA": { + "RD": { "type": "boolean" }, - "RD": { + "RA": { "type": "boolean" }, + "Z": { + "type": "integer" + }, + "answers": { + "type": "string" + }, "rejected": { "type": "boolean" }, - "qclass_name": { + /* + * Conn log support + * https://www.bro.org/sphinx/scripts/base/protocols/conn/main.bro.html#type-Conn::Info + * + * Notable Fields + * Field: proto + * Notes: Field exists in the DNS, Conn, DPD, and Notice logs + * + * Field: duration + * Notes: Field exists in the Conn and Files logs + * + * Field: local_orig + * Notes: Field exists in the Conn and Files logs + */ + "service": { "type": "string", "index": "not_analyzed" }, - "proto": { + "duration": { + "type": "float" + }, + "orig_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "resp_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "conn_state": { "type": "string", "index": "not_analyzed" }, - "rcode": { - "type": "integer" + "local_orig": { + "type": "boolean" }, - "rcode_name": { + "local_resp": { "type": "string", "index": "not_analyzed" }, - "trans_id": { + "missed_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "history": { + "type": "string", + "index": "not_analyzed" + }, + "orig_pkts": { + "type": "long", + "index": "not_analyzed" + }, + "orig_ip_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "resp_pkts": { + "type": "long", + "index": "not_analyzed" + }, + "resp_ip_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "tunnel_parents": { + "type": "string", + "index": "not_analyzed" + }, + /* + * DPD log support + * https://www.bro.org/sphinx-git/scripts/base/frameworks/dpd/main.bro.html#type-DPD::Info + * + * Notable Fields + * Field: proto + * Notes: Field exists in the DNS, Conn, DPD, and Notice logs + */ + "analyzer": { + "type": "string", + "index": "not_analyzed" + }, + "failure_reason": { + "type": "string", + "index": "not_analyzed" + }, + /* + * FTP log support + * https://www.bro.org/sphinx/scripts/base/protocols/ftp/info.bro.html#type-FTP::Info + * + * Notable Fields + * Field: password + * Notes: Field exists in the HTTP and FTP logs + * + * Field: capture_password + * Notes: Field exists in the HTTP and FTP logs + * + * Field: mime_type + * Notes: Field exists in the FTP and Files logs + * + * Field: fuid + * Notes: Field exists in the FTP and Notice logs + */ + "user": { + "type": "string", + "index": "not_analyzed" + }, + "command": { + "type": "string", + "index": "not_analyzed" + }, + "arg": { + "type": "string", + "analyzer": "simple" + }, + "mime_type": { + "type": "string", + "analyzer": "simple" + }, + "file_size": { + "type": "long" + }, + "reply_code": { "type": "integer" }, - "Z": { + "reply_msg": { + "type": "string", + "index": "not_analyzed" + }, + "data_channel:passive": { + "type": "boolean" + }, + "data_channel:orig_h": { + "type": "ip" + }, + "data_channel:resp_h": { + "type": "ip" + }, + "data_channel:resp_p": { "type": "integer" }, - "qclass": { + "cwd": { + "type": "string", + "analyzer": "simple" + }, + "passive": { + "type": "boolean" + }, + "fuid": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Files log support + * https://www.bro.org/sphinx/scripts/base/frameworks/files/main.bro.html#type-Files::Info + * + * Notable Fields + * Field: tx_hosts + * Notes: Metron rewrites this to "ip_src_addr" + * + * Field: rx_hosts + * Notes: Metron rewrites this to "ip_dst_addr" + * + * Field: mime_type + * Notes: Field exists in the FTP and Files logs + */ + "conn_uids": { + "type": "string", + "analyzer": "simple" + }, + "source": { + "type": "string", + "index": "not_analyzed" + }, + "depth": { "type": "integer" }, - "qtype": { + "analyzers": { + "type": "string", + "analyzer": "simple" + }, + "filename": { + "type": "string", + "index": "not_analyzed" + }, + "is_orig": { + "type": "boolean" + }, + "seen_bytes": { + "type": "long" + }, + "total_bytes": { + "type": "long" + }, + "missing_bytes": { + "type": "long" + }, + "overflow_bytes": { + "type": "long" + }, + "timedout": { + "type": "boolean" + }, + "parent_fuid": { + "type": "string", + "index": "not_analyzed" + }, + "md5": { + "type": "string", + "index": "not_analyzed" + }, + "sha1": { + "type": "string", + "index": "not_analyzed" + }, + "sha256": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Known::CertInfo log support + * https://www.bro.org/sphinx/scripts/policy/protocols/ssl/known-certs.bro.html#type-Known::CertsInfo + * + * Notable Fields + * Field: subject + * Notes: Field exists in the Known::CertInfo and SMTP logs + */ + "port_num": { "type": "integer" }, - "qtype_name": { + "subject": { + "type": "string", + "analyzer": "simple" + }, + "issuer_subject": { + "type": "string", + "analyzer": "simple" + }, + "serial": { + "type": "string", + "index": "not_analyzed" + }, + /* + * SMTP log support + * https://www.bro.org/sphinx/scripts/base/protocols/smtp/main.bro.html#type-SMTP::Info + * + * Notable Fields + * Field: subject + * Notes: Field exists in the Known::CertInfo and SMTP logs + */ + "helo": { + "type": "string", + "analyzer": "simple" + }, + "mailfrom": { + "type": "string", + "analyzer": "simple" + }, + "rcptto": { + "type": "string", + "analyzer": "simple" + }, + "date": { + "type": "string", + "index": "not_analyzed" + }, + "from": { + "type": "string", + "analyzer": "simple" + }, + "to": { + "type": "string", + "analyzer": "simple" + }, + "reply_to": { + "type": "string", + "analyzer": "simple" + }, + "msg_id": { + "type": "string", + "index": "not_analyzed" + }, + "in_reply_to": { + "type": "string", + "index": "not_analyzed" + }, + "x_originating_ip": { + "type": "ip" + }, + "first_received": { + "type": "string", + "analyzer": "simple" + }, + "second_received": { + "type": "string", + "analyzer": "simple" + }, + "last_reply": { + "type": "string", + "analyzer": "simple" + }, + "path": { + "type": "string", + "index": "not_analyzed" + }, + "tls": { + "type": "boolean" + }, + "fuids": { + "type": "string", + "index": "not_analyzed" + }, + "is_webmail": { + "type": "boolean" + }, + /* + * SSL log support + * https://www.bro.org/sphinx/scripts/base/protocols/ssl/main.bro.html#type-SSL::Info + * + * Notable Fields + * Field: version + * Notes: Field exists in the HTTP, SSL, and SSH logs + */ + "cipher": { + "type": "string", + "index": "not_analyzed" + }, + "curve": { + "type": "string", + "index": "not_analyzed" + }, + "server_name": { + "type": "string", + "index": "not_analyzed" + }, + "resumed": { + "type": "boolean" + }, + "last_alert": { + "type": "string", + "index": "not_analyzed" + }, + "next_protocol": { + "type": "string", + "index": "not_analyzed" + }, + "established": { + "type": "boolean" + }, + /* + * Weird log support + * https://www.bro.org/sphinx/scripts/base/frameworks/notice/weird.bro.html#type-Weird::Info + */ + "name": { + "type": "string", + "index": "not_analyzed" + }, + "addl": { + "type": "string", + "index": "not_analyzed" + }, + "notice": { + "type": "boolean" + }, + "peer": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Notice log support + * https://www.bro.org/sphinx/scripts/base/frameworks/notice/main.bro.html#type-Notice::Info + * + * Notable Fields + * Field: fuid + * Notes: Field exists in the FTP and Notice logs + * + * Field: proto + * Notes: Field exists in the DNS, Conn, DPD, and Notice logs + */ + "file_mime_type": { + "type": "string", + "index": "not_analyzed" + }, + "file_desc": { + "type": "string", + "index": "not_analyzed" + }, + "note": { + "type": "string", + "index": "not_analyzed" + }, + "msg": { + "type": "string", + "index": "not_analyzed" + }, + "sub": { + "type": "string", + "index": "not_analyzed" + }, + "src": { + "type": "ip" + }, + "dst": { + "type": "ip" + }, + "p": { + "type": "integer", + "index": "not_analyzed" + }, + "n": { + "type": "integer", + "index": "not_analyzed" + }, + "src_peer": { + "type": "ip" + }, + "peer_descr": { + "type": "string", + "index": "not_analyzed" + }, + "actions": { + "type": "string", + "index": "not_analyzed" + }, + "suppress_for": { + "type": "double", + "index": "not_analyzed" + }, + "dropped": { + "type": "boolean" + }, + /* + * DHCP log support + * https://www.bro.org/sphinx/scripts/base/protocols/dhcp/main.bro.html#type-DHCP::Info + * + * Notable Fields + * Field: trans_id + * Notes: Field exists in the DNS and DHCP logs + * + * Field: mac + * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs + */ + "mac": { + "type": "string", + "index": "not_analyzed" + }, + "assigned_ip": { + "type": "ip" + }, + "lease_time": { + "type": "float", + "index": "not_analyzed" + }, + /* + * SSH log support + * https://www.bro.org/sphinx/scripts/base/protocols/ssh/main.bro.html#type-SSH::Info + * + * Notable Fields + * Field: version + * Notes: Field exists in the HTTP, SSL, and SSH logs + */ + "auth_success": { + "type": "boolean" + }, + "auth_attempts": { + "type": "integer", + "index": "not_analyzed" + }, + "direction": { + "type": "string", + "index": "not_analyzed" + }, + "client": { + "type": "string", + "index": "not_analyzed" + }, + "server": { + "type": "string", + "index": "not_analyzed" + }, + "cipher_alg": { + "type": "string", + "index": "not_analyzed" + }, + "mac_alg": { + "type": "string", + "index": "not_analyzed" + }, + "compression_alg": { + "type": "string", + "index": "not_analyzed" + }, + "kex_alg": { + "type": "string", + "index": "not_analyzed" + }, + "host_key_alg": { + "type": "string", + "index": "not_analyzed" + }, + "host_key": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Software log support + * https://www.bro.org/sphinx/scripts/base/frameworks/software/main.bro.html#type-Software::Info + * + * Notable Fields + * Field: host + * Notes: Field exists in the HTTP and Software logs + */ + "host_p": { + "type": "integer", + "index": "not_analyzed" + }, + "software_type": { + "type": "string", + "index": "not_analyzed" + }, + "version:major": { + "type": "string", + "index": "not_analyzed" + }, + "version:minor": { + "type": "string", + "index": "not_analyzed" + }, + "version:minor2": { + "type": "string", + "index": "not_analyzed" + }, + "version:minor3": { + "type": "string", + "index": "not_analyzed" + }, + "version:addl": { + "type": "string", + "index": "not_analyzed" + }, + "unparsed_version": { + "type": "string", + "analyzer": "simple" + }, + /* + * RADIUS log support + * https://www.bro.org/sphinx/scripts/base/protocols/radius/main.bro.html#type-RADIUS::Info + * + * Notable Fields + * Field: username + * Notes: Field exists in the HTTP and RADIUS logs + * + * Field: mac + * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs + */ + "remote_ip": { + "type": "ip" + }, + "connect_info": { + "type": "string", + "index": "not_analyzed" + }, + "result": { + "type": "string", + "index": "not_analyzed" + }, + /* + * X509 log support + * https://www.bro.org/sphinx/scripts/base/files/x509/main.bro.html#type-X509::Info + * + * Notable Fields + * Field: id + * Notes: In other bro records, the id field is of type conn_id, so it is + * expanded before being logged into 4 fields, all of which are addressed + * under the "Widely-used Bro fields" section of this template. In X509 + * logs, however, id is a string to identify the certificate file id. + */ + "id": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:version": { + "type": "integer", + "index": "not_analyzed" + }, + "certificate:serial": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:subject": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:issuer": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:not_valid_before": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:not_valid_after": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:key_alg": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:sig_alg": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:key_type": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:key_length": { + "type": "integer", + "index": "not_analyzed" + }, + "certificate:exponent": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:curve": { + "type": "string", + "index": "not_analyzed" + }, + "san:dns": { + "type": "string", + "index": "not_analyzed" + }, + "san:uri": { + "type": "string", + "index": "not_analyzed" + }, + "san:email": { + "type": "string", + "index": "not_analyzed" + }, + "san:ip": { + "type": "string", + "index": "not_analyzed" + }, + "basic_constraints:ca": { + "type": "boolean" + }, + "basic_constraints:path_len": { + "type": "integer", + "index": "not_analyzed" + }, + /* + * Known::DevicesInfo log support + * https://www.bro.org/sphinx/scripts/policy/misc/known-devices.bro.html#type-Known::DevicesInfo + * + * Notable Fields + * Field: mac + * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs + */ + "dhcp_host_name": { "type": "string", "index": "not_analyzed" } diff --git a/metron-platform/metron-integration-test/src/main/sample/data/bro/parsed/BroExampleParsed b/metron-platform/metron-integration-test/src/main/sample/data/bro/parsed/BroExampleParsed index d2775200b3..b1d3102175 100644 --- a/metron-platform/metron-integration-test/src/main/sample/data/bro/parsed/BroExampleParsed +++ b/metron-platform/metron-integration-test/src/main/sample/data/bro/parsed/BroExampleParsed @@ -8,3 +8,20 @@ {"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"CTo78A11g7CYbbOHvj","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:CTo78A11g7CYbbOHvj resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd"],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"} {"TTLs":[3600.0,289.0,14.0],"qclass_name":"C_INTERNET","bro_timestamp":"1402308259.609","qtype_name":"AAAA","ip_dst_port":53,"qtype":28,"rejected":false,"answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"trans_id":62418,"uid":"CuJT272SKaJSuqO0Ia","protocol":"dns","original_string":"DNS | AA:true TTLs:[3600.0,289.0,14.0] qclass_name:C_INTERNET id.orig_p:33976 qtype_name:AAAA qtype:28 rejected:false id.resp_p:53 query:www.cisco.com answers:[\"www.cisco.com.akadns.net\",\"origin-www.cisco.com\",\"2001:420:1201:2::a\"] trans_id:62418 rcode:0 rcode_name:NOERROR TC:false RA:true uid:CuJT272SKaJSuqO0Ia RD:true proto:udp id.orig_h:10.122.196.204 Z:0 qclass:1 ts:1402308259.609 id.resp_h:144.254.71.184","ip_dst_addr":"144.254.71.184","Z":0,"ip_src_addr":"10.122.196.204","qclass":1,"timestamp":1402308259609,"AA":true,"query":"www.cisco.com","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","RD":true,"ip_src_port":33976,"proto":"udp","guid":"this-is-random-uuid-will-be-36-chars"} {"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:KIRAN resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:10.122.196.204 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"10.122.196.204","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd"],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"} +{"bro_timestamp":"1440447880.931272","resp_pkts":1,"resp_ip_bytes":48,"ip_dst_port":1812,"orig_bytes":75,"orig_ip_bytes":103,"orig_pkts":1,"missed_bytes":0,"history":"Dd","tunnel_parents":[],"source.type":"bro","duration":1.001459,"uid":"CWxtRHnBTbldHnmGh","protocol":"conn","resp_bytes":20,"original_string":"CONN | id.orig_p:52178 resp_pkts:1 resp_ip_bytes:48 orig_bytes:75 id.resp_p:1812 orig_ip_bytes:103 orig_pkts:1 missed_bytes:0 history:Dd tunnel_parents:[] duration:1.001459 uid:CWxtRHnBTbldHnmGh resp_bytes:20 service:radius conn_state:SF proto:udp id.orig_h:127.0.0.1 ts:1440447880.931272 id.resp_h:127.0.0.1","ip_dst_addr":"127.0.0.1","ip_src_port":52178,"service":"radius","conn_state":"SF","proto":"udp","guid":"4a92fe07-8f9d-4092-83c3-0d4e37c92d29","ip_src_addr":"127.0.0.1","timestamp":1440447880931} +{"bro_timestamp":"1440447904.122012","resp_pkts":0,"resp_ip_bytes":0,"ip_dst_port":1812,"orig_bytes":225,"orig_ip_bytes":309,"orig_pkts":3,"missed_bytes":0,"history":"D","tunnel_parents":[],"source.type":"bro","duration":10.008839,"uid":"CK2Oivhlh0ovRcYx","protocol":"conn","resp_bytes":0,"original_string":"CONN | id.orig_p:62956 resp_pkts:0 resp_ip_bytes:0 orig_bytes:225 id.resp_p:1812 orig_ip_bytes:309 orig_pkts:3 missed_bytes:0 history:D tunnel_parents:[] duration:10.008839 uid:CK2Oivhlh0ovRcYx resp_bytes:0 service:radius conn_state:S0 proto:udp id.orig_h:127.0.0.1 ts:1440447904.122012 id.resp_h:127.0.0.1","ip_dst_addr":"127.0.0.1","ip_src_port":62956,"service":"radius","conn_state":"S0","proto":"udp","guid":"9e4952e0-6dd3-4487-b5fa-299b9433c381","ip_src_addr":"127.0.0.1","timestamp":1440447904122} +{"bro_timestamp":"1440448190.335333","resp_pkts":1,"resp_ip_bytes":99,"ip_dst_port":1812,"orig_bytes":75,"orig_ip_bytes":103,"orig_pkts":1,"missed_bytes":0,"history":"Dd","tunnel_parents":[],"source.type":"bro","duration":5.17E-4,"uid":"CX6mcO38sO7dkDxK55","protocol":"conn","resp_bytes":71,"original_string":"CONN | id.orig_p:53127 resp_pkts:1 resp_ip_bytes:99 orig_bytes:75 id.resp_p:1812 orig_ip_bytes:103 orig_pkts:1 missed_bytes:0 history:Dd tunnel_parents:[] duration:0.000517 uid:CX6mcO38sO7dkDxK55 resp_bytes:71 service:radius conn_state:SF proto:udp id.orig_h:127.0.0.1 ts:1440448190.335333 id.resp_h:127.0.0.1","ip_dst_addr":"127.0.0.1","ip_src_port":53127,"service":"radius","conn_state":"SF","proto":"udp","guid":"bc1af1bf-5b1c-4829-b574-3243670fd448","ip_src_addr":"127.0.0.1","timestamp":1440448190335} +{"bro_timestamp":"1216702277.477596","ip_dst_port":80,"failure_reason":"not a http reply line","source.type":"bro","uid":"C4O50B3WAUCb2Yw29j","protocol":"dpd","original_string":"DPD | uid:C4O50B3WAUCb2Yw29j id.orig_p:33348 analyzer:HTTP id.resp_p:80 proto:tcp id.orig_h:192.168.15.4 failure_reason:not a http reply line ts:1216702277.477596 id.resp_h:66.33.212.43","ip_dst_addr":"66.33.212.43","ip_src_port":33348,"analyzer":"HTTP","proto":"tcp","guid":"b03d9d34-4a39-4e68-8b21-08bdd532ae07","ip_src_addr":"192.168.15.4","timestamp":1216702277477} +{"bro_timestamp":"1166289883.160785","ip_dst_port":21,"reply_msg":"Entering Passive Mode (192,168,0,193,28,86)","data_channel.orig_h":"192.168.0.114","data_channel.passive":true,"data_channel.resp_p":7254,"command":"PASV","source.type":"bro","uid":"ClOsCM3BUs3saPsD2c","password":"","protocol":"ftp","original_string":"FTP | id.orig_p:1137 id.resp_p:21 reply_msg:Entering Passive Mode (192,168,0,193,28,86) data_channel.orig_h:192.168.0.114 data_channel.passive:true data_channel.resp_p:7254 command:PASV uid:ClOsCM3BUs3saPsD2c password: data_channel.resp_h:192.168.0.193 id.orig_h:192.168.0.114 user:csanders reply_code:227 ts:1166289883.160785 id.resp_h:192.168.0.193","ip_dst_addr":"192.168.0.193","ip_src_port":1137,"data_channel.resp_h":"192.168.0.193","guid":"4b0c4cda-28ee-404e-b966-036bc7f638ff","user":"csanders","ip_src_addr":"192.168.0.114","reply_code":227,"timestamp":1166289883160} +{"bro_timestamp":"1216706983.387664","timedout":true,"source":"HTTP","is_orig":false,"overflow_bytes":0,"source.type":"bro","duration":30.701792,"protocol":"files","depth":0,"original_string":"FILES | timedout:true rx_hosts:[\"192.168.15.4\"] source:HTTP is_orig:false tx_hosts:[\"216.113.185.92\"] overflow_bytes:0 duration:30.701792 depth:0 analyzers:[\"MD5\",\"SHA1\"] fuid:FnEYba9VPOcC41c1 conn_uids:[\"CLWqoN1IA9MB8Ru9i3\"] seen_bytes:0 missing_bytes:3384 ts:1216706983.387664","ip_dst_addr":"192.168.15.4","analyzers":["MD5","SHA1"],"guid":"7b7148a0-f484-4450-97a3-29493e1c7360","fuid":"FnEYba9VPOcC41c1","conn_uids":["CLWqoN1IA9MB8Ru9i3"],"seen_bytes":0,"missing_bytes":3384,"ip_src_addr":"216.113.185.92","timestamp":1216706983387} +{"bro_timestamp":"1216706999.34818","protocol":"known_certs","original_string":"KNOWN_CERTS | issuer_subject:CN=VeriSign Class 3 Secure Server CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)05,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US serial:24A2DD82DC52358E7F0C6AF6135F3B32 subject:CN=nexus.passport.com,OU=MSN Passport,O=Microsoft,L=Redmond,ST=Washington,C=US port_num:443 host:65.54.179.216 ts:1216706999.34818","issuer_subject":"CN=VeriSign Class 3 Secure Server CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)05,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US","serial":"24A2DD82DC52358E7F0C6AF6135F3B32","subject":"CN=nexus.passport.com,OU=MSN Passport,O=Microsoft,L=Redmond,ST=Washington,C=US","port_num":443,"host":"65.54.179.216","guid":"76fe881c-3ed7-4477-a870-f5381577e4ae","timestamp":1216706999348,"source.type":"bro"} +{"bro_timestamp":"1258568036.57884","ip_dst_port":25,"source.type":"bro","helo":"M57Terry","uid":"ChR6254RrWbrxiGsd7","path":["192.168.1.1","192.168.1.105"],"trans_depth":1,"protocol":"smtp","original_string":"SMTP | id.orig_p:49353 id.resp_p:25 helo:M57Terry uid:ChR6254RrWbrxiGsd7 path:[\"192.168.1.1\",\"192.168.1.105\"] trans_depth:1 is_webmail:false last_reply:220 2.0.0 Ready to start TLS id.orig_h:192.168.1.105 tls:true fuids:[] ts:1258568036.57884 id.resp_h:192.168.1.1","ip_dst_addr":"192.168.1.1","ip_src_port":49353,"is_webmail":false,"last_reply":"220 2.0.0 Ready to start TLS","guid":"9a3d1e86-7d25-4426-b2af-6ab5be1e607f","tls":true,"fuids":[],"ip_src_addr":"192.168.1.105","timestamp":1258568036578} +{"cipher":"TLS_RSA_WITH_RC4_128_MD5","established":true,"server_name":"login.live.com","bro_timestamp":"1216706999.444925","client_cert_chain_fuids":[],"ip_dst_port":443,"subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","cert_chain_fuids":["FkYBO41LPAXxh44KFk","FPrzYN1SuBqHflXZId","FZ71xF13r5XVSam1z1"],"version":"TLSv10","issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US","source.type":"bro","uid":"CVrS2IBW8gukBClA8","protocol":"ssl","original_string":"SSL | cipher:TLS_RSA_WITH_RC4_128_MD5 established:true server_name:login.live.com id.orig_p:36532 client_cert_chain_fuids:[] subject:CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553 id.resp_p:443 cert_chain_fuids:[\"FkYBO41LPAXxh44KFk\",\"FPrzYN1SuBqHflXZId\",\"FZ71xF13r5XVSam1z1\"] version:TLSv10 issuer:CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US uid:CVrS2IBW8gukBClA8 id.orig_h:192.168.15.4 validation_status:unable to get local issuer certificate resumed:false ts:1216706999.444925 id.resp_h:65.54.186.47","ip_dst_addr":"65.54.186.47","ip_src_port":36532,"guid":"1bff79d0-7b86-43de-b5ec-132bb62f4339","validation_status":"unable to get local issuer certificate","resumed":false,"ip_src_addr":"192.168.15.4","timestamp":1216706999444} +{"bro_timestamp":"1216706981.177382","ip_dst_port":80,"source.type":"bro","uid":"Cfxxnt3m0v9SEf5XQ7","protocol":"weird","original_string":"WEIRD | uid:Cfxxnt3m0v9SEf5XQ7 id.orig_p:36446 peer:bro id.resp_p:80 name:unescaped_special_URI_char id.orig_h:192.168.15.4 ts:1216706981.177382 id.resp_h:66.151.146.194 notice:false","ip_dst_addr":"66.151.146.194","ip_src_port":36446,"peer":"bro","name":"unescaped_special_URI_char","guid":"fa2d1068-ca33-4962-b9ab-902605ea3e14","ip_src_addr":"192.168.15.4","notice":false,"timestamp":1216706981177} +{"msg":"SSL certificate validation failed with (unable to get local issuer certificate)","suppress_for":3600.0,"note":"SSL::Invalid_Server_Cert","sub":"CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US","bro_timestamp":"1216706377.196728","dst":"74.125.19.104","ip_dst_port":443,"src":"192.168.15.4","dropped":false,"peer_descr":"bro","source.type":"bro","p":443,"uid":"CNHQmp1mNiZHdAf5Ce","protocol":"notice","original_string":"NOTICE | msg:SSL certificate validation failed with (unable to get local issuer certificate) suppress_for:3600.0 note:SSL::Invalid_Server_Cert sub:CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US id.orig_p:35736 dst:74.125.19.104 src:192.168.15.4 id.resp_p:443 dropped:false peer_descr:bro p:443 uid:CNHQmp1mNiZHdAf5Ce proto:tcp id.orig_h:192.168.15.4 actions:[\"Notice::ACTION_LOG\"] ts:1216706377.196728 id.resp_h:74.125.19.104","ip_dst_addr":"74.125.19.104","ip_src_port":35736,"proto":"tcp","guid":"31e56b6a-48fd-4605-81ec-b0586006f7d7","actions":["Notice::ACTION_LOG"],"ip_src_addr":"192.168.15.4","timestamp":1216706377196} +{"bro_timestamp":"1258567562.944638","ip_dst_port":67,"trans_id":418901490,"assigned_ip":"192.168.1.103","mac":"00:0b:db:63:5b:d4","source.type":"bro","uid":"CSiO9f3y8Uyu0XprAi","protocol":"dhcp","original_string":"DHCP | uid:CSiO9f3y8Uyu0XprAi id.orig_p:68 lease_time:3564.0 id.resp_p:67 id.orig_h:192.168.1.103 trans_id:418901490 assigned_ip:192.168.1.103 mac:00:0b:db:63:5b:d4 ts:1258567562.944638 id.resp_h:192.168.1.1","ip_dst_addr":"192.168.1.1","ip_src_port":68,"lease_time":3564.0,"guid":"0d2ed5dc-f44c-4d37-b286-7b9f40da420a","ip_src_addr":"192.168.1.103","timestamp":1258567562944} +{"kex_alg":"diffie-hellman-group-exchange-sha256","server":"SSH-2.0-OpenSSH_5.8p1 Debian-7ubuntu1","mac_alg":"hmac-md5","bro_timestamp":"1320435930.914196","auth_success":false,"ip_dst_port":22,"host_key_alg":"ssh-rsa","compression_alg":"none","version":2,"source.type":"bro","uid":"CyrWKo1E1rRywjbOAk","host_key":"87:11:46:da:89:c5:2b:d9:6b:ee:e0:44:7e:73:80:f8","protocol":"ssh","original_string":"SSH | kex_alg:diffie-hellman-group-exchange-sha256 server:SSH-2.0-OpenSSH_5.8p1 Debian-7ubuntu1 mac_alg:hmac-md5 id.orig_p:58435 auth_success:false id.resp_p:22 host_key_alg:ssh-rsa compression_alg:none version:2 uid:CyrWKo1E1rRywjbOAk host_key:87:11:46:da:89:c5:2b:d9:6b:ee:e0:44:7e:73:80:f8 cipher_alg:aes128-ctr client:SSH-2.0-OpenSSH_5.6 id.orig_h:172.16.238.1 ts:1320435930.914196 id.resp_h:172.16.238.136","ip_dst_addr":"172.16.238.136","ip_src_port":58435,"cipher_alg":"aes128-ctr","client":"SSH-2.0-OpenSSH_5.6","guid":"8aebc887-4090-4807-8d65-e841f52b6177","ip_src_addr":"172.16.238.1","timestamp":1320435930914} +{"bro_timestamp":"1320435464.768382","software_type":"SSH::SERVER","source.type":"bro","unparsed_version":"OpenSSH_5.3","protocol":"software","host_p":22,"original_string":"SOFTWARE | unparsed_version:OpenSSH_5.3 host_p:22 host:172.16.238.168 name:OpenSSH software_type:SSH::SERVER version.major:5 version.minor:3 ts:1320435464.768382","host":"172.16.238.168","name":"OpenSSH","guid":"ad3d1b4b-ffad-4416-be0f-7df08587ccb5","version.major":5,"version.minor":3,"timestamp":1320435464768} +{"bro_timestamp":"1440447766.441298","ip_dst_port":1812,"source.type":"bro","result":"failed","uid":"CqF4zGzBOXFjTWqHh","protocol":"radius","original_string":"RADIUS | result:failed uid:CqF4zGzBOXFjTWqHh id.orig_p:53031 id.resp_p:1812 id.orig_h:127.0.0.1 ts:1440447766.441298 id.resp_h:127.0.0.1 username:steve","ip_dst_addr":"127.0.0.1","ip_src_port":53031,"guid":"b029735a-3e98-45a0-b8da-232967a34085","ip_src_addr":"127.0.0.1","username":"steve","timestamp":1440447766441} +{"certificate.key_length":1024,"bro_timestamp":"1216706999.661483","certificate.sig_alg":"sha1WithRSAEncryption","certificate.not_valid_before":1.2138336E9,"certificate.key_type":"rsa","basic_constraints.ca":false,"certificate.key_alg":"rsaEncryption","certificate.exponent":"65537","source.type":"bro","protocol":"x509","original_string":"X509 | certificate.key_length:1024 certificate.sig_alg:sha1WithRSAEncryption certificate.not_valid_before:1213833600.0 certificate.key_type:rsa basic_constraints.ca:false certificate.key_alg:rsaEncryption certificate.exponent:65537 certificate.version:3 certificate.subject:CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553 id:FkYBO41LPAXxh44KFk certificate.not_valid_after:1248134399.0 certificate.serial:6905C4A47CFDBF9DBC98DACE38835FB8 certificate.issuer:CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US ts:1216706999.661483","certificate.version":3,"certificate.subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","guid":"578eac04-9024-49ab-828d-e25f01c33c82","id":"FkYBO41LPAXxh44KFk","certificate.not_valid_after":1.248134399E9,"certificate.serial":"6905C4A47CFDBF9DBC98DACE38835FB8","certificate.issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US","timestamp":1216706999661} +{"bro_timestamp":"1258531221.486539","protocol":"known_devices","original_string":"KNOWN_DEVICES | dhcp_host_name:m57-jo mac:00:0b:db:63:58:a6 ts:1258531221.486539","dhcp_host_name":"m57-jo","guid":"e7a216d8-3623-4dea-af78-01da8c5e0bc5","mac":"00:0b:db:63:58:a6","timestamp":1258531221486,"source.type":"bro"} diff --git a/metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput b/metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput index 4ff0699783..5c88714609 100644 --- a/metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput +++ b/metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput @@ -8,3 +8,20 @@ {"http":{"ts":1402307733.473,"uid":"CTo78A11g7CYbbOHvj","id.orig_h":"192.249.113.37","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"method":"GET","host":"www.cisco.com","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} {"dns":{"ts":1402308259.609,"uid":"CuJT272SKaJSuqO0Ia","id.orig_h":"10.122.196.204","id.orig_p":33976,"id.resp_h":"144.254.71.184","id.resp_p":53,"proto":"udp","trans_id":62418,"query":"www.cisco.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"TTLs":[3600.0,289.0,14.0],"rejected":false}} {"http":{"ts":1402307733.473,"uid":"KIRAN","id.orig_h":"10.122.196.204","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"method":"GET","host":"www.cisco.com","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} +{"conn": {"ts":1440447880.931272,"uid":"CWxtRHnBTbldHnmGh","id.orig_h":"127.0.0.1","id.orig_p":52178,"id.resp_h":"127.0.0.1","id.resp_p":1812,"proto":"udp","service":"radius","duration":1.001459,"orig_bytes":75,"resp_bytes":20,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":48,"tunnel_parents":[]}} +{"conn": {"ts":1440447904.122012,"uid":"CK2Oivhlh0ovRcYx","id.orig_h":"127.0.0.1","id.orig_p":62956,"id.resp_h":"127.0.0.1","id.resp_p":1812,"proto":"udp","service":"radius","duration":10.008839,"orig_bytes":225,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":3,"orig_ip_bytes":309,"resp_pkts":0,"resp_ip_bytes":0,"tunnel_parents":[]}} +{"conn": {"ts":1440448190.335333,"uid":"CX6mcO38sO7dkDxK55","id.orig_h":"127.0.0.1","id.orig_p":53127,"id.resp_h":"127.0.0.1","id.resp_p":1812,"proto":"udp","service":"radius","duration":0.000517,"orig_bytes":75,"resp_bytes":71,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":99,"tunnel_parents":[]}} +{"dpd": {"ts":1216702277.477596,"uid":"C4O50B3WAUCb2Yw29j","id.orig_h":"192.168.15.4","id.orig_p":33348,"id.resp_h":"66.33.212.43","id.resp_p":80,"proto":"tcp","analyzer":"HTTP","failure_reason":"not a http reply line"}} +{"ftp": {"ts":1166289883.160785,"uid":"ClOsCM3BUs3saPsD2c","id.orig_h":"192.168.0.114","id.orig_p":1137,"id.resp_h":"192.168.0.193","id.resp_p":21,"user":"csanders","password":"","command":"PASV","reply_code":227,"reply_msg":"Entering Passive Mode (192,168,0,193,28,86)","data_channel.passive":true,"data_channel.orig_h":"192.168.0.114","data_channel.resp_h":"192.168.0.193","data_channel.resp_p":7254}} +{"files": {"ts":1216706983.387664,"fuid":"FnEYba9VPOcC41c1","tx_hosts":["216.113.185.92"],"rx_hosts":["192.168.15.4"],"conn_uids":["CLWqoN1IA9MB8Ru9i3"],"source":"HTTP","depth":0,"analyzers":["MD5","SHA1"],"duration":30.701792,"is_orig":false,"seen_bytes":0,"missing_bytes":3384,"overflow_bytes":0,"timedout":true}} +{"known_certs": {"ts":1216706999.34818,"host":"65.54.179.216","port_num":443,"subject":"CN=nexus.passport.com,OU=MSN Passport,O=Microsoft,L=Redmond,ST=Washington,C=US","issuer_subject":"CN=VeriSign Class 3 Secure Server CA,OU=Terms of use at https://www.verisign.com/rpa (c)05,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US","serial":"24A2DD82DC52358E7F0C6AF6135F3B32"}} +{"smtp": {"ts":1258568036.57884,"uid":"ChR6254RrWbrxiGsd7","id.orig_h":"192.168.1.105","id.orig_p":49353,"id.resp_h":"192.168.1.1","id.resp_p":25,"trans_depth":1,"helo":"M57Terry","last_reply":"220 2.0.0 Ready to start TLS","path":["192.168.1.1","192.168.1.105"],"tls":true,"fuids":[],"is_webmail":false}} +{"ssl": {"ts":1216706999.444925,"uid":"CVrS2IBW8gukBClA8","id.orig_h":"192.168.15.4","id.orig_p":36532,"id.resp_h":"65.54.186.47","id.resp_p":443,"version":"TLSv10","cipher":"TLS_RSA_WITH_RC4_128_MD5","server_name":"login.live.com","resumed":false,"established":true,"cert_chain_fuids":["FkYBO41LPAXxh44KFk","FPrzYN1SuBqHflXZId","FZ71xF13r5XVSam1z1"],"client_cert_chain_fuids":[],"subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\u005c, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US","validation_status":"unable to get local issuer certificate"}} +{"weird": {"ts":1216706981.177382,"uid":"Cfxxnt3m0v9SEf5XQ7","id.orig_h":"192.168.15.4","id.orig_p":36446,"id.resp_h":"66.151.146.194","id.resp_p":80,"name":"unescaped_special_URI_char","notice":false,"peer":"bro"}} +{"notice": {"ts":1216706377.196728,"uid":"CNHQmp1mNiZHdAf5Ce","id.orig_h":"192.168.15.4","id.orig_p":35736,"id.resp_h":"74.125.19.104","id.resp_p":443,"proto":"tcp","note":"SSL::Invalid_Server_Cert","msg":"SSL certificate validation failed with (unable to get local issuer certificate)","sub":"CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US","src":"192.168.15.4","dst":"74.125.19.104","p":443,"peer_descr":"bro","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0,"dropped":false}} +{"dhcp": {"ts":1258567562.944638,"uid":"CSiO9f3y8Uyu0XprAi","id.orig_h":"192.168.1.103","id.orig_p":68,"id.resp_h":"192.168.1.1","id.resp_p":67,"mac":"00:0b:db:63:5b:d4","assigned_ip":"192.168.1.103","lease_time":3564.0,"trans_id":418901490}} +{"ssh": {"ts":1320435930.914196,"uid":"CyrWKo1E1rRywjbOAk","id.orig_h":"172.16.238.1","id.orig_p":58435,"id.resp_h":"172.16.238.136","id.resp_p":22,"version":2,"auth_success":false,"client":"SSH-2.0-OpenSSH_5.6","server":"SSH-2.0-OpenSSH_5.8p1 Debian-7ubuntu1","cipher_alg":"aes128-ctr","mac_alg":"hmac-md5","compression_alg":"none","kex_alg":"diffie-hellman-group-exchange-sha256","host_key_alg":"ssh-rsa","host_key":"87:11:46:da:89:c5:2b:d9:6b:ee:e0:44:7e:73:80:f8"}} +{"software": {"ts":1320435464.768382,"host":"172.16.238.168","host_p":22,"software_type":"SSH::SERVER","name":"OpenSSH","version.major":5,"version.minor":3,"unparsed_version":"OpenSSH_5.3"}} +{"radius": {"ts":1440447766.441298,"uid":"CqF4zGzBOXFjTWqHh","id.orig_h":"127.0.0.1","id.orig_p":53031,"id.resp_h":"127.0.0.1","id.resp_p":1812,"username":"steve","result":"failed"}} +{"x509": {"ts":1216706999.661483,"id":"FkYBO41LPAXxh44KFk","certificate.version":3,"certificate.serial":"6905C4A47CFDBF9DBC98DACE38835FB8","certificate.subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\u005c, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","certificate.issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US","certificate.not_valid_before":1213833600.0,"certificate.not_valid_after":1248134399.0,"certificate.key_alg":"rsaEncryption","certificate.sig_alg":"sha1WithRSAEncryption","certificate.key_type":"rsa","certificate.key_length":1024,"certificate.exponent":"65537","basic_constraints.ca":false}} +{"known_devices": {"ts":1258531221.486539,"mac":"00:0b:db:63:58:a6","dhcp_host_name":"m57-jo"}} diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java index bed1f256dc..d910d1296b 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java @@ -38,7 +38,7 @@ public class JSONCleaner implements Serializable { * @param jsonString * @return * @throws ParseException - * Takes a json String as input and removes any Special Chars (^ a-z A-Z 0-9) in the keys + * Takes a json String as input and modifies the keys to remove any characters other than . _ a-z A-Z or 0-9 */ @SuppressWarnings({"unchecked","rawtypes"}) public JSONObject clean(String jsonString) throws ParseException diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java index 7b09746bcf..aa60d1f521 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java @@ -17,6 +17,7 @@ */ package org.apache.metron.parsers.bro; +import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.lang3.tuple.Pair; import org.apache.log4j.Level; import org.apache.metron.test.utils.UnitTestHelper; @@ -66,43 +67,92 @@ public void testDecimalFormatAssumptions() { } } + /** + * { + * "ts":1449511228.474, + * "uid":"CFgSLp4HgsGqXnNjZi", + * "id.orig_h":"104.130.172.191", + * "id.orig_p":33893, + * "id.resp_h":"69.20.0.164", + * "id.resp_p":53, + * "proto":"udp", + * "trans_id":3514, + * "rcode":3, + * "rcode_name":"NXDOMAIN", + * "AA":false, + * "TC":false, + * "RD":false, + * "RA":false, + * "Z":0, + * "rejected":false, + * "sensor":"cloudbro", + * "type":"dns" + * } + */ + @Multiline + public final static String unwrappedBroMessage; + @Test public void testUnwrappedBroMessage() throws ParseException { - String rawMessage = "{\"timestamp\":1449511228.474,\"uid\":\"CFgSLp4HgsGqXnNjZi\",\"source_ip\":\"104.130.172.191\",\"source_port\":33893,\"dest_ip\":\"69.20.0.164\",\"dest_port\":53,\"proto\":\"udp\",\"trans_id\":3514,\"rcode\":3,\"rcode_name\":\"NXDOMAIN\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"rejected\":false,\"sensor\":\"cloudbro\",\"type\":\"dns\"}"; - - JSONObject rawJson = (JSONObject)jsonParser.parse(rawMessage); - - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject rawJson = (JSONObject)jsonParser.parse(unwrappedBroMessage); + JSONObject broJson = broParser.parse(unwrappedBroMessage.getBytes()).get(0); - String expectedBroTimestamp = "1449511228.474"; + String expectedBroTimestamp = "1449511228.474"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1449511228474"; - Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); - Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("source_ip").toString()); - Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("dest_ip").toString()); - Assert.assertEquals(broJson.get("ip_src_port"), rawJson.get("source_port")); - Assert.assertEquals(broJson.get("ip_dst_port"), rawJson.get("dest_port")); + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port"), rawJson.get("id.orig_p")); + Assert.assertEquals(broJson.get("ip_dst_port"), rawJson.get("id.resp_p")); Assert.assertEquals(broJson.get("uid").toString(), rawJson.get("uid").toString()); Assert.assertEquals(broJson.get("trans_id").toString(), rawJson.get("trans_id").toString()); Assert.assertEquals(broJson.get("sensor").toString(), rawJson.get("sensor").toString()); - Assert.assertEquals(broJson.get("protocol").toString(), rawJson.get("type").toString()); + Assert.assertEquals(broJson.get("type").toString(), rawJson.get("type").toString()); Assert.assertEquals(broJson.get("rcode").toString(), rawJson.get("rcode").toString()); Assert.assertEquals(broJson.get("rcode_name").toString(), rawJson.get("rcode_name").toString()); - Assert.assertTrue(broJson.get("original_string").toString().startsWith("DNS")); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("DNS")); } + /** + * { + * "http": { + * "ts":1402307733.473, + * "uid":"CTo78A11g7CYbbOHvj", + * "id.orig_h":"192.249.113.37", + * "id.orig_p":58808, + * "id.resp_h":"72.163.4.161", + * "id.resp_p":80, + * "trans_depth":1, + * "method":"GET", + * "host":"www.cisco.com", + * "uri":"/", + * "user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3", + * "request_body_len":0, + * "response_body_len":25523, + * "status_code":200, + * "status_msg":"OK", + * "tags":[], + * "resp_fuids":["FJDyMC15lxUn5ngPfd"], + * "resp_mime_types":["text/html"] + * } + * } + */ + @Multiline + public final static String httpBroMessage; + @SuppressWarnings("rawtypes") @Test public void testHttpBroMessage() throws ParseException { - String rawMessage = "{\"http\":{\"ts\":1402307733.473,\"uid\":\"CTo78A11g7CYbbOHvj\",\"id.orig_h\":\"192.249.113.37\",\"id.orig_p\":58808,\"id.resp_h\":\"72.163.4.161\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"www.cisco.com\",\"uri\":\"/\",\"user_agent\":\"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3\",\"request_body_len\":0,\"response_body_len\":25523,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FJDyMC15lxUn5ngPfd\"],\"resp_mime_types\":[\"text/html\"]}}"; - - Map rawMessageMap = (Map) jsonParser.parse(rawMessage); + Map rawMessageMap = (Map) jsonParser.parse(httpBroMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(httpBroMessage.getBytes()).get(0); String expectedBroTimestamp = "1402307733.473"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); - String expectedTimestamp = "1402307733473"; + String expectedTimestamp = "1402307733473"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); @@ -114,6 +164,8 @@ public void testHttpBroMessage() throws ParseException { Assert.assertEquals(broJson.get("method").toString(), rawJson.get("method").toString()); Assert.assertEquals(broJson.get("host").toString(), rawJson.get("host").toString()); Assert.assertEquals(broJson.get("resp_mime_types").toString(), rawJson.get("resp_mime_types").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("HTTP")); } /** @@ -178,14 +230,40 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException } } + /** + * { + * "http": { + * "ts":1457149494.166991, + * "uid":"CTo78A11g7CYbbOHvj", + * "id.orig_h":"192.249.113.37", + * "id.orig_p":58808, + * "id.resp_h":"72.163.4.161", + * "id.resp_p":80, + * "trans_depth":1, + * "method":"GET", + * "host":"www.cisco.com", + * "uri":"/", + * "user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3", + * "request_body_len":0, + * "response_body_len":25523, + * "status_code":200, + * "status_msg":"OK", + * "tags":[], + * "resp_fuids":["FJDyMC15lxUn5ngPfd"], + * "resp_mime_types":["text/html"] + * } + * } + */ + @Multiline + public final static String httpBroDecimalMessage; + @SuppressWarnings("rawtypes") @Test - public void testHttpDecimalBroMessage() throws ParseException { - String rawMessage = "{\"http\":{\"ts\":1457149494.166991,\"uid\":\"CTo78A11g7CYbbOHvj\",\"id.orig_h\":\"192.249.113.37\",\"id.orig_p\":58808,\"id.resp_h\":\"72.163.4.161\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"www.cisco.com\",\"uri\":\"/\",\"user_agent\":\"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3\",\"request_body_len\":0,\"response_body_len\":25523,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FJDyMC15lxUn5ngPfd\"],\"resp_mime_types\":[\"text/html\"]}}"; - Map rawMessageMap = (Map) jsonParser.parse(rawMessage); + public void testHttpBroDecimalMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(httpBroDecimalMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(httpBroDecimalMessage.getBytes()).get(0); String expectedBroTimestamp = "1457149494.166991"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1457149494166"; @@ -200,18 +278,50 @@ public void testHttpDecimalBroMessage() throws ParseException { Assert.assertEquals(broJson.get("method").toString(), rawJson.get("method").toString()); Assert.assertEquals(broJson.get("host").toString(), rawJson.get("host").toString()); Assert.assertEquals(broJson.get("resp_mime_types").toString(), rawJson.get("resp_mime_types").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("HTTP")); } + /** + * { + * "dns": { + * "ts":1402308259.609, + * "uid":"CuJT272SKaJSuqO0Ia", + * "id.orig_h":"10.122.196.204", + * "id.orig_p":33976, + * "id.resp_h":"144.254.71.184", + * "id.resp_p":53, + * "proto":"udp", + * "trans_id":62418, + * "query":"www.cisco.com", + * "qclass":1, + * "qclass_name":"C_INTERNET", + * "qtype":28, + * "qtype_name":"AAAA", + * "rcode":0, + * "rcode_name":"NOERROR", + * "AA":true, + * "TC":false, + * "RD":true, + * "RA":true, + * "Z":0, + * "answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"], + * "TTLs":[3600.0,289.0,14.0], + * "rejected":false + * } + * } + */ + @Multiline + public final static String dnsBroMessage; + @SuppressWarnings("rawtypes") @Test public void testDnsBroMessage() throws ParseException { - String rawMessage = "{\"dns\":{\"ts\":1402308259.609,\"uid\":\"CuJT272SKaJSuqO0Ia\",\"id.orig_h\":\"10.122.196.204\",\"id.orig_p\":33976,\"id.resp_h\":\"144.254.71.184\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":62418,\"query\":\"www.cisco.com\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":28,\"qtype_name\":\"AAAA\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":true,\"TC\":false,\"RD\":true,\"RA\":true,\"Z\":0,\"answers\":[\"www.cisco.com.akadns.net\",\"origin-www.cisco.com\",\"2001:420:1201:2::a\"],\"TTLs\":[3600.0,289.0,14.0],\"rejected\":false}}"; - - Map rawMessageMap = (Map) jsonParser.parse(rawMessage); + Map rawMessageMap = (Map) jsonParser.parse(dnsBroMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(dnsBroMessage.getBytes()).get(0); String expectedBroTimestamp = "1402308259.609"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1402308259609"; @@ -224,17 +334,43 @@ public void testDnsBroMessage() throws ParseException { Assert.assertEquals(broJson.get("qtype").toString(), rawJson.get("qtype").toString()); Assert.assertEquals(broJson.get("trans_id").toString(), rawJson.get("trans_id").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("DNS")); } + /** + * { + * "files": { + * "analyzers": ["X509","MD5","SHA1"], + * "conn_uids":["C4tygJ3qxJBEJEBCeh"], + * "depth": 0, + * "duration": 0.0, + * "fuid":"FZEBC33VySG0nHSoO9", + * "is_orig": false, + * "local_orig": false, + * "md5": "eba37166385e3ef42464ed9752e99f1b", + * "missing_bytes": 0, + * "overflow_bytes": 0, + * "rx_hosts": ["10.220.15.205"], + * "seen_bytes": 1136, + * "sha1": "73e42686657aece354fbf685712361658f2f4357", + * "source": "SSL", + * "timedout": false, + * "ts": 1425845251.334, + * "tx_hosts": ["68.171.237.7"] + * } + * } + */ + @Multiline + public final static String filesBroMessage; + @SuppressWarnings("rawtypes") @Test public void testFilesBroMessage() throws ParseException { - String rawMessage = "{\"files\":{\"analyzers\": [\"X509\",\"MD5\",\"SHA1\"],\"conn_uids\":[\"C4tygJ3qxJBEJEBCeh\"],\"depth\": 0,\"duration\": 0.0,\"fuid\":\"FZEBC33VySG0nHSoO9\",\"is_orig\": false,\"local_orig\": false,\"md5\": \"eba37166385e3ef42464ed9752e99f1b\",\"missing_bytes\": 0,\"overflow_bytes\": 0,\"rx_hosts\": [\"10.220.15.205\"],\"seen_bytes\": 1136,\"sha1\": \"73e42686657aece354fbf685712361658f2f4357\",\"source\": \"SSL\",\"timedout\": false,\"ts\": 1425845251.334,\"tx_hosts\": [\"68.171.237.7\"]}}"; - - Map rawMessageMap = (Map) jsonParser.parse(rawMessage); + Map rawMessageMap = (Map) jsonParser.parse(filesBroMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(filesBroMessage.getBytes()).get(0); String expectedBroTimestamp = "1425845251.334"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1425845251334"; @@ -246,22 +382,798 @@ public void testFilesBroMessage() throws ParseException { Assert.assertEquals(broJson.get("fuid").toString(), rawJson.get("fuid").toString()); Assert.assertEquals(broJson.get("md5").toString(), rawJson.get("md5").toString()); Assert.assertEquals(broJson.get("analyzers").toString(), rawJson.get("analyzers").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("FILES")); } + /** + * { + * "conn": { + * "ts":1166289883.163553, + * "uid":"CTKCLy1z4C9U8OqU0c", + * "id.orig_h":"192.168.0.114", + * "id.orig_p":1140, + * "id.resp_h":"192.168.0.193", + * "id.resp_p":7254, + * "proto":"tcp", + * "service":"ftp-data", + * "duration":0.006635, + * "orig_bytes":0, + * "resp_bytes":5808, + * "conn_state":"S1", + * "missed_bytes":0, + * "history":"ShAd", + * "orig_pkts":3, + * "orig_ip_bytes":128, + * "resp_pkts":5, + * "resp_ip_bytes":6016, + * "tunnel_parents":[] + * } + * } + */ + @Multiline + public final static String connBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testConnBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(connBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(connBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1166289883.163553"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1166289883163"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("proto").toString(), rawJson.get("proto").toString()); + Assert.assertEquals(broJson.get("service").toString(), rawJson.get("service").toString()); + Assert.assertEquals(broJson.get("duration").toString(), rawJson.get("duration").toString()); + Assert.assertEquals(broJson.get("orig_bytes").toString(), rawJson.get("orig_bytes").toString()); + Assert.assertEquals(broJson.get("resp_bytes").toString(), rawJson.get("resp_bytes").toString()); + Assert.assertEquals(broJson.get("conn_state").toString(), rawJson.get("conn_state").toString()); + Assert.assertEquals(broJson.get("missed_bytes").toString(), rawJson.get("missed_bytes").toString()); + Assert.assertEquals(broJson.get("history").toString(), rawJson.get("history").toString()); + Assert.assertEquals(broJson.get("orig_pkts").toString(), rawJson.get("orig_pkts").toString()); + Assert.assertEquals(broJson.get("orig_ip_bytes").toString(), rawJson.get("orig_ip_bytes").toString()); + Assert.assertEquals(broJson.get("resp_pkts").toString(), rawJson.get("resp_pkts").toString()); + Assert.assertEquals(broJson.get("resp_ip_bytes").toString(), rawJson.get("resp_ip_bytes").toString()); + Assert.assertEquals(broJson.get("tunnel_parents").toString(), rawJson.get("tunnel_parents").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("CONN")); + } + + /** + * { + * "dpd": { + * "ts":1216704078.712276, + * "uid":"CwlB8d119WPanz63J", + * "id.orig_h":"192.168.15.4", + * "id.orig_p":34508, + * "id.resp_h":"66.33.212.43", + * "id.resp_p":80, + * "proto":"tcp", + * "analyzer":"HTTP", + * "failure_reason":"not a http reply line" + * } + * } + */ + @Multiline + public final static String dpdBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testDpdBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(dpdBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(dpdBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1216704078.712276"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1216704078712"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("proto").toString(), rawJson.get("proto").toString()); + Assert.assertEquals(broJson.get("analyzer").toString(), rawJson.get("analyzer").toString()); + Assert.assertEquals(broJson.get("failure_reason").toString(), rawJson.get("failure_reason").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("DPD")); + } + + /** + * { + * "ftp": { + * "ts":1166289883.164645, + * "uid":"CuVhX03cii8zrjrtva", + * "id.orig_h":"192.168.0.114", + * "id.orig_p":1137, + * "id.resp_h":"192.168.0.193", + * "id.resp_p":21, + * "user":"csanders", + * "password":"", + * "command":"RETR", + * "arg":"ftp://192.168.0.193/Music.mp3", + * "mime_type":"", + * "file_size":192, + * "reply_code":150, + * "reply_msg":"Data connection accepted from 192.168.0.114:1140; transfer starting for Music.mp3 (4980924 bytes).", + * "fuid":"FlS6Jg1aNdsBxNn9Bf" + * } + * } + */ + @Multiline + public final static String ftpBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testFtpBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(ftpBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(ftpBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1166289883.164645"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1166289883164"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("user").toString(), rawJson.get("user").toString()); + Assert.assertEquals(broJson.get("password").toString(), rawJson.get("password").toString()); + Assert.assertEquals(broJson.get("command").toString(), rawJson.get("command").toString()); + Assert.assertEquals(broJson.get("arg").toString(), rawJson.get("arg").toString()); + Assert.assertEquals(broJson.get("mime_type").toString(), rawJson.get("mime_type").toString()); + Assert.assertEquals(broJson.get("file_size").toString(), rawJson.get("file_size").toString()); + Assert.assertEquals(broJson.get("reply_code").toString(), rawJson.get("reply_code").toString()); + Assert.assertEquals(broJson.get("reply_msg").toString(), rawJson.get("reply_msg").toString()); + Assert.assertEquals(broJson.get("fuid").toString(), rawJson.get("fuid").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("FTP")); + } + + /** + * { + * "known_certs": { + * "ts":1216706999.896836, + * "host":"65.54.186.47", + * "port_num":443, + * "subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\u005c, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553", + * "issuer_subject":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US", + * "serial":"6905C4A47CFDBF9DBC98DACE38835FB8" + * } + * } + */ + @Multiline + public final static String knownCertsBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testKnownCertsBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(knownCertsBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(knownCertsBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1216706999.896836"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1216706999896"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("host").toString(), rawJson.get("host").toString()); + Assert.assertEquals(broJson.get("port_num").toString(), rawJson.get("port_num").toString()); + Assert.assertEquals(broJson.get("subject").toString(), rawJson.get("subject").toString()); + Assert.assertEquals(broJson.get("issuer_subject").toString(), rawJson.get("issuer_subject").toString()); + Assert.assertEquals(broJson.get("serial").toString(), rawJson.get("serial").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("KNOWN_CERTS")); + } + + /** + * { + * "smtp": { + * "ts":1258568059.130219, + * "uid":"CMeLem2ouYvV8fzUp9", + * "id.orig_h":"192.168.1.103", + * "id.orig_p":1836, + * "id.resp_h":"192.168.1.1", + * "id.resp_p":25, + * "trans_depth":1, + * "helo":"m57pat", + * "last_reply":"220 2.0.0 Ready to start TLS", + * "path":["192.168.1.1","192.168.1.103"], + * "tls":true, + * "fuids":[], + * "is_webmail":false + * } + * } + */ + @Multiline + public final static String smtpBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testSmtpBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(smtpBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(smtpBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1258568059.130219"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1258568059130"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("trans_depth").toString(), rawJson.get("trans_depth").toString()); + Assert.assertEquals(broJson.get("helo").toString(), rawJson.get("helo").toString()); + Assert.assertEquals(broJson.get("last_reply").toString(), rawJson.get("last_reply").toString()); + Assert.assertEquals(broJson.get("path").toString(), rawJson.get("path").toString()); + Assert.assertEquals(broJson.get("tls").toString(), rawJson.get("tls").toString()); + Assert.assertEquals(broJson.get("fuids").toString(), rawJson.get("fuids").toString()); + Assert.assertEquals(broJson.get("is_webmail").toString(), rawJson.get("is_webmail").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("SMTP")); + } + + /** + * { + * "ssl": { + * "ts":1216706999.444925, + * "uid":"Chy3Ge1k0IceXK4Di", + * "id.orig_h":"192.168.15.4", + * "id.orig_p":36532, + * "id.resp_h":"65.54.186.47", + * "id.resp_p":443, + * "version":"TLSv10", + * "cipher":"TLS_RSA_WITH_RC4_128_MD5", + * "server_name":"login.live.com", + * "resumed":false, + * "established":true, + * "cert_chain_fuids":["FkYBO41LPAXxh44KFk","FPrzYN1SuBqHflXZId","FZ71xF13r5XVSam1z1"], + * "client_cert_chain_fuids":[], + * "subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\u005c, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553", + * "issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US", + * "validation_status":"unable to get local issuer certificate" + * } + * } + */ + @Multiline + public final static String sslBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testSslBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(sslBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(sslBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1216706999.444925"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1216706999444"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("version").toString(), rawJson.get("version").toString()); + Assert.assertEquals(broJson.get("cipher").toString(), rawJson.get("cipher").toString()); + Assert.assertEquals(broJson.get("server_name").toString(), rawJson.get("server_name").toString()); + Assert.assertEquals(broJson.get("resumed").toString(), rawJson.get("resumed").toString()); + Assert.assertEquals(broJson.get("established").toString(), rawJson.get("established").toString()); + Assert.assertEquals(broJson.get("cert_chain_fuids").toString(), rawJson.get("cert_chain_fuids").toString()); + Assert.assertEquals(broJson.get("client_cert_chain_fuids").toString(), rawJson.get("client_cert_chain_fuids").toString()); + Assert.assertEquals(broJson.get("subject").toString(), rawJson.get("subject").toString()); + Assert.assertEquals(broJson.get("issuer").toString(), rawJson.get("issuer").toString()); + Assert.assertEquals(broJson.get("validation_status").toString(), rawJson.get("validation_status").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("SSL")); + } + + /** + * { + * "weird": { + * "ts":1216706886.239896, + * "uid":"CLSluk42pqbExeZQFl", + * "id.orig_h":"192.168.15.4", + * "id.orig_p":36336, + * "id.resp_h":"66.151.146.194", + * "id.resp_p":80, + * "name":"unescaped_special_URI_char", + * "notice":false, + * "peer":"bro" + * } + * } + */ + @Multiline + public final static String weirdBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testWeirdBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(weirdBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(weirdBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1216706886.239896"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1216706886239"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("name").toString(), rawJson.get("name").toString()); + Assert.assertEquals(broJson.get("notice").toString(), rawJson.get("notice").toString()); + Assert.assertEquals(broJson.get("peer").toString(), rawJson.get("peer").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("WEIRD")); + } + + /** + * { + * "notice": { + * "ts":1216706377.196728, + * "uid":"CgpsTT28ZTiuSEsfVi", + * "id.orig_h":"192.168.15.4", + * "id.orig_p":35736, + * "id.resp_h":"74.125.19.104", + * "id.resp_p":443, + * "proto":"tcp", + * "note":"SSL::Invalid_Server_Cert", + * "msg":"SSL certificate validation failed with (unable to get local issuer certificate)", + * "sub":"CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US", + * "src":"192.168.15.4", + * "dst":"74.125.19.104", + * "p":443, + * "peer_descr":"bro", + * "actions":["Notice::ACTION_LOG"], + * "suppress_for":3600.0, + * "dropped":false + * } + * } + */ + @Multiline + public final static String noticeBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testNoticeBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(noticeBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(noticeBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1216706377.196728"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1216706377196"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("proto").toString(), rawJson.get("proto").toString()); + Assert.assertEquals(broJson.get("note").toString(), rawJson.get("note").toString()); + Assert.assertEquals(broJson.get("msg").toString(), rawJson.get("msg").toString()); + Assert.assertEquals(broJson.get("sub").toString(), rawJson.get("sub").toString()); + Assert.assertEquals(broJson.get("src").toString(), rawJson.get("src").toString()); + Assert.assertEquals(broJson.get("dst").toString(), rawJson.get("dst").toString()); + Assert.assertEquals(broJson.get("p").toString(), rawJson.get("p").toString()); + Assert.assertEquals(broJson.get("peer_descr").toString(), rawJson.get("peer_descr").toString()); + Assert.assertEquals(broJson.get("actions").toString(), rawJson.get("actions").toString()); + Assert.assertEquals(broJson.get("suppress_for").toString(), rawJson.get("suppress_for").toString()); + Assert.assertEquals(broJson.get("dropped").toString(), rawJson.get("dropped").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("NOTICE")); + } + + /** + * { + * "dhcp": { + * "ts":1258567562.944638, + * "uid":"C8rZDh400N68UV9Ulj", + * "id.orig_h":"192.168.1.103", + * "id.orig_p":68, + * "id.resp_h":"192.168.1.1", + * "id.resp_p":67, + * "mac":"00:0b:db:63:5b:d4", + * "assigned_ip":"192.168.1.103", + * "lease_time":3564.0, + * "trans_id":418901490 + * } + * } + */ + @Multiline + public final static String dhcpBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testDhcpBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(dhcpBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(dhcpBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1258567562.944638"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1258567562944"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("mac").toString(), rawJson.get("mac").toString()); + Assert.assertEquals(broJson.get("assigned_ip").toString(), rawJson.get("assigned_ip").toString()); + Assert.assertEquals(broJson.get("lease_time").toString(), rawJson.get("lease_time").toString()); + Assert.assertEquals(broJson.get("trans_id").toString(), rawJson.get("trans_id").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("DHCP")); + } + + /** + * { + * "ssh": { + * "ts":1320435870.747967, + * "uid":"CSbqud1LKhRqlJiLDg", + * "id.orig_h":"172.16.238.1", + * "id.orig_p":58429, + * "id.resp_h":"172.16.238.136", + * "id.resp_p":22, + * "version":2, + * "auth_success":false, + * "client":"SSH-2.0-OpenSSH_5.6", + * "server":"SSH-2.0-OpenSSH_5.8p1 Debian-7ubuntu1", + * "cipher_alg":"aes128-ctr", + * "mac_alg":"hmac-md5", + * "compression_alg":"none", + * "kex_alg":"diffie-hellman-group-exchange-sha256", + * "host_key_alg":"ssh-rsa", + * "host_key":"87:11:46:da:89:c5:2b:d9:6b:ee:e0:44:7e:73:80:f8" + * } + * } + */ + @Multiline + public final static String sshBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testSshBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(sshBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(sshBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1320435870.747967"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1320435870747"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("version").toString(), rawJson.get("version").toString()); + Assert.assertEquals(broJson.get("auth_success").toString(), rawJson.get("auth_success").toString()); + Assert.assertEquals(broJson.get("client").toString(), rawJson.get("client").toString()); + Assert.assertEquals(broJson.get("server").toString(), rawJson.get("server").toString()); + Assert.assertEquals(broJson.get("cipher_alg").toString(), rawJson.get("cipher_alg").toString()); + Assert.assertEquals(broJson.get("mac_alg").toString(), rawJson.get("mac_alg").toString()); + Assert.assertEquals(broJson.get("compression_alg").toString(), rawJson.get("compression_alg").toString()); + Assert.assertEquals(broJson.get("kex_alg").toString(), rawJson.get("kex_alg").toString()); + Assert.assertEquals(broJson.get("host_key_alg").toString(), rawJson.get("host_key_alg").toString()); + Assert.assertEquals(broJson.get("host_key").toString(), rawJson.get("host_key").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("SSH")); + } + + /** + * { + * "software": { + * "ts":1216707079.49066, + * "host":"38.102.35.231", + * "host_p":80, + * "software_type":"HTTP::SERVER", + * "name":"lighttpd", + * "version.major":1, + * "version.minor":4, + * "version.minor2":18, + * "unparsed_version":"lighttpd/1.4.18" + * } + * } + */ + @Multiline + public final static String softwareBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testSoftwareBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(softwareBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(softwareBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1216707079.49066"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1216707079490"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("host").toString(), rawJson.get("host").toString()); + Assert.assertEquals(broJson.get("host_p").toString(), rawJson.get("host_p").toString()); + Assert.assertEquals(broJson.get("software_type").toString(), rawJson.get("software_type").toString()); + Assert.assertEquals(broJson.get("name").toString(), rawJson.get("name").toString()); + Assert.assertEquals(broJson.get("version.major").toString(), rawJson.get("version.major").toString()); + Assert.assertEquals(broJson.get("version.minor").toString(), rawJson.get("version.minor").toString()); + Assert.assertEquals(broJson.get("version.minor2").toString(), rawJson.get("version.minor2").toString()); + Assert.assertEquals(broJson.get("unparsed_version").toString(), rawJson.get("unparsed_version").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("SOFTWARE")); + } + + /** + * { + * "software": { + * "ts":1216707079.518447, + * "host":"72.21.202.98", + * "host_p":80, + * "software_type":"HTTP::SERVER", + * "name":"AmazonS3", + * "unparsed_version":"AmazonS3" + * } + * } + */ + @Multiline + public final static String softwareBroMessage2; + + @SuppressWarnings("rawtypes") + @Test + public void testSoftwareBroMessage2() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(softwareBroMessage2); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(softwareBroMessage2.getBytes()).get(0); + String expectedBroTimestamp = "1216707079.518447"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1216707079518"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("host").toString(), rawJson.get("host").toString()); + Assert.assertEquals(broJson.get("host_p").toString(), rawJson.get("host_p").toString()); + Assert.assertEquals(broJson.get("software_type").toString(), rawJson.get("software_type").toString()); + Assert.assertEquals(broJson.get("name").toString(), rawJson.get("name").toString()); + Assert.assertEquals(broJson.get("unparsed_version").toString(), rawJson.get("unparsed_version").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("SOFTWARE")); + } + + /** + * { + * "radius": { + * "ts":1440447766.441298, + * "uid":"Cfvksv4SEJJiqFobPj", + * "id.orig_h":"127.0.0.1", + * "id.orig_p":53031, + * "id.resp_h":"127.0.0.1", + * "id.resp_p":1812, + * "username":"steve", + * "result":"failed" + * } + * } + */ + @Multiline + public final static String radiusBroMessageFailed; + + @SuppressWarnings("rawtypes") + @Test + public void testRadiusBroMessageFailed() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(radiusBroMessageFailed); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(radiusBroMessageFailed.getBytes()).get(0); + String expectedBroTimestamp = "1440447766.441298"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1440447766441"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("username").toString(), rawJson.get("username").toString()); + Assert.assertEquals(broJson.get("result").toString(), rawJson.get("result").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("RADIUS")); + } + + /** + * { + * "radius": { + * "ts":1440447839.947956, + * "uid":"CHb5MF3GTmyPniTage", + * "id.orig_h":"127.0.0.1", + * "id.orig_p":65443, + * "id.resp_h":"127.0.0.1", + * "id.resp_p":1812, + * "username":"steve", + * "result":"success" + * } + * } + */ + @Multiline + public final static String radiusBroMessageSuccess; + + @SuppressWarnings("rawtypes") + @Test + public void testRadiusBroMessageSuccess() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(radiusBroMessageSuccess); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(radiusBroMessageSuccess.getBytes()).get(0); + String expectedBroTimestamp = "1440447839.947956"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1440447839947"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("username").toString(), rawJson.get("username").toString()); + Assert.assertEquals(broJson.get("result").toString(), rawJson.get("result").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("RADIUS")); + } + + /** + * { + * "x509": { + * "ts":1216706999.661483, + * "id":"FPrzYN1SuBqHflXZId", + * "certificate.version":3, + * "certificate.serial":"5B7759C61784E15EC727C0329529286B", + * "certificate.subject":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US","certificate.issuer":"CN=VeriSign Class 3 Public Primary Certification Authority - G5,OU=(c) 2006 VeriSign\u005c, Inc. - For authorized use only,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US", + * "certificate.not_valid_before":1162944000.0, + * "certificate.not_valid_after":1478563199.0, + * "certificate.key_alg":"rsaEncryption", + * "certificate.sig_alg":"sha1WithRSAEncryption", + * "certificate.key_type":"rsa", + * "certificate.key_length":2048, + * "certificate.exponent":"65537", + * "basic_constraints.ca":true, + * "basic_constraints.path_len":0 + * } + * } + */ + @Multiline + public final static String x509BroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testX509BroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(x509BroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(x509BroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1216706999.661483"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1216706999661"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("id").toString(), rawJson.get("id").toString()); + Assert.assertEquals(broJson.get("certificate.version").toString(), rawJson.get("certificate.version").toString()); + Assert.assertEquals(broJson.get("certificate.serial").toString(), rawJson.get("certificate.serial").toString()); + Assert.assertEquals(broJson.get("certificate.subject").toString(), rawJson.get("certificate.subject").toString()); + Assert.assertEquals(broJson.get("certificate.issuer").toString(), rawJson.get("certificate.issuer").toString()); + Assert.assertEquals(broJson.get("certificate.not_valid_before").toString(), rawJson.get("certificate.not_valid_before").toString()); + Assert.assertEquals(broJson.get("certificate.not_valid_after").toString(), rawJson.get("certificate.not_valid_after").toString()); + Assert.assertEquals(broJson.get("certificate.key_alg").toString(), rawJson.get("certificate.key_alg").toString()); + Assert.assertEquals(broJson.get("certificate.sig_alg").toString(), rawJson.get("certificate.sig_alg").toString()); + Assert.assertEquals(broJson.get("certificate.key_type").toString(), rawJson.get("certificate.key_type").toString()); + Assert.assertEquals(broJson.get("certificate.key_length").toString(), rawJson.get("certificate.key_length").toString()); + Assert.assertEquals(broJson.get("certificate.exponent").toString(), rawJson.get("certificate.exponent").toString()); + Assert.assertEquals(broJson.get("basic_constraints.ca").toString(), rawJson.get("basic_constraints.ca").toString()); + Assert.assertEquals(broJson.get("basic_constraints.path_len").toString(), rawJson.get("basic_constraints.path_len").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("X509")); + } + + /** + * { + * "known_devices": { + * "ts":1258532046.693816, + * "mac":"00:0b:db:4f:6b:10", + * "dhcp_host_name":"m57-charlie" + * } + * } + */ + @Multiline + public final static String knownDevicesBroMessage; + + @SuppressWarnings("rawtypes") + @Test + public void testKnownDevicesBroMessage() throws ParseException { + Map rawMessageMap = (Map) jsonParser.parse(knownDevicesBroMessage); + JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); + + JSONObject broJson = broParser.parse(knownDevicesBroMessage.getBytes()).get(0); + String expectedBroTimestamp = "1258532046.693816"; + Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); + String expectedTimestamp = "1258532046693"; + Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); + Assert.assertTrue(broJson.get("original_string").toString().startsWith(rawMessageMap.keySet().iterator().next().toString().toUpperCase())); + + Assert.assertEquals(broJson.get("mac").toString(), rawJson.get("mac").toString()); + Assert.assertEquals(broJson.get("dhcp_host_name").toString(), rawJson.get("dhcp_host_name").toString()); + + Assert.assertTrue(broJson.get("original_string").toString().startsWith("KNOWN_DEVICES")); + } + + /** + * { + * "ht*tp": { + * "ts":1402307733.473, + * "uid":"CTo78A11g7CYbbOHvj", + * "id.orig_h":"192.249.113.37", + * "id.orig_p":58808, + * "id.resp_h":"72.163.4.161", + * "id.resp_p":80, + * "trans_depth":1, + * "method":"GET", + * "host":"www.cisco.com", + * "uri":"/", + * "user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3", + * "request_body_len":0, + * "response_body_len":25523, + * "status_code":200, + * "status_msg":"OK", + * "tags":[], + * "resp_fuids":["FJDyMC15lxUn5ngPfd"], + * "resp_mime_types":["text/html"] + * } + * } + */ + @Multiline + public final static String protocolKeyCleanedUp; + @SuppressWarnings("rawtypes") @Test public void testProtocolKeyCleanedUp() throws ParseException { - String rawMessage = "{\"ht*tp\":{\"ts\":1402307733.473,\"uid\":\"CTo78A11g7CYbbOHvj\",\"id.orig_h\":\"192.249.113.37\",\"id.orig_p\":58808,\"id.resp_h\":\"72.163.4.161\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"www.cisco.com\",\"uri\":\"/\",\"user_agent\":\"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3\",\"request_body_len\":0,\"response_body_len\":25523,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FJDyMC15lxUn5ngPfd\"],\"resp_mime_types\":[\"text/html\"]}}"; - - Map rawMessageMap = (Map) jsonParser.parse(rawMessage); + Map rawMessageMap = (Map) jsonParser.parse(protocolKeyCleanedUp); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); - JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0); + JSONObject broJson = broParser.parse(protocolKeyCleanedUp.getBytes()).get(0); String expectedBroTimestamp = "1402307733.473"; Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp); String expectedTimestamp = "1402307733473"; Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp); - Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString()); + Assert.assertEquals(broJson.get("ip_dst_addr").toString(), rawJson.get("id.resp_h").toString()); + Assert.assertEquals(broJson.get("ip_src_port").toString(), rawJson.get("id.orig_p").toString()); + Assert.assertEquals(broJson.get("ip_dst_port").toString(), rawJson.get("id.resp_p").toString()); Assert.assertTrue(broJson.get("original_string").toString().startsWith("HTTP")); }