diff --git a/debian/control b/debian/control index d4dd932c..007fb483 100644 --- a/debian/control +++ b/debian/control @@ -149,6 +149,19 @@ Description: Vyatta dataplane sample plugin library A sample pipeline plugin for the Vyatta dataplane that illustrates how a plugin can work. +Package: vyatta-dataplane-flowstat-plugin +Section: non-free/net +Architecture: any +Depends: vyatta-dataplane | vyatta-dataplane-test, + ${misc:Depends}, + ${perl:Depends}, + ${python3:Depends}, + ${shlibs:Depends}, + python3 +Description: Vyatta dataplane plugin library. + A pipeline plugin for Vyatta dataplane that exports + flow statistics. + Package: libvyattafal-dev Section: non-free/libdevel Architecture: any diff --git a/debian/rules b/debian/rules index 81697dc9..8fda8179 100755 --- a/debian/rules +++ b/debian/rules @@ -76,3 +76,4 @@ override_dh_gencontrol: override_dh_systemd_enable: dh_systemd_enable --name=vyatta-dataplane dh_systemd_enable --name=proc-xen + dh_systemd_enable --name=flowstatd diff --git a/debian/vyatta-dataplane-flowstat-plugin.flowstatd.service b/debian/vyatta-dataplane-flowstat-plugin.flowstatd.service new file mode 100644 index 00000000..1564cee0 --- /dev/null +++ b/debian/vyatta-dataplane-flowstat-plugin.flowstatd.service @@ -0,0 +1,10 @@ +[Unit] +Description=Flowstat Daemon + +[Service] +Type=simple +ExecStart=/opt/vyatta/bin/flowstatd +Restart=on-failure + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/debian/vyatta-dataplane-flowstat-plugin.install b/debian/vyatta-dataplane-flowstat-plugin.install new file mode 100644 index 00000000..c2a24246 --- /dev/null +++ b/debian/vyatta-dataplane-flowstat-plugin.install @@ -0,0 +1,5 @@ +usr/lib/*/vyatta-dataplane/pipeline/plugins/flowstat_plugin.so +usr/share/flowstat_plugin/protobuf/* /usr/share/vyatta-dataplane/protobuf +usr/share/perl5/flowstat_plugin/proto/* /usr/share/perl5/vyatta/proto +src/pipeline/nodes/flowstat/yang/*.yang usr/share/configd/yang +src/pipeline/nodes/flowstat/scripts/* opt/vyatta/bin diff --git a/include/dp_session.h b/include/dp_session.h index 8201da2e..1aaddf15 100644 --- a/include/dp_session.h +++ b/include/dp_session.h @@ -65,6 +65,63 @@ enum dp_session_state { SESSION_STATE_CLOSED, } __attribute__ ((__packed__)); +/** + * Session attribute. + */ +enum dp_session_attr { + SESSION_ATTR_BYTES_IN = 1, + SESSION_ATTR_PKTS_IN = (1 << 1), + SESSION_ATTR_PROTOCOL = (1 << 2), + SESSION_ATTR_TCP_FLAGS = (1 << 3), + SESSION_ATTR_L4_SRC_PORT = (1 << 4), + SESSION_ATTR_IPV4_SRC_ADDR = (1 << 5), + SESSION_ATTR_L4_DST_PORT = (1 << 6), + SESSION_ATTR_IPV4_DST_ADDR = (1 << 7), + SESSION_ATTR_CREATE_TIME = (1 << 8), + SESSION_ATTR_BYTES_OUT = (1 << 9), + SESSION_ATTR_PKTS_OUT = (1 << 10), + SESSION_ATTR_IF_NAME = (1 << 11), + SESSION_ATTR_DPI = (1 << 12), +}; + +#define SESSION_ATTR_ALL 0xffffffff +#define SESSION_ATTR_SENTRY (SESSION_ATTR_L4_SRC_PORT \ + | SESSION_ATTR_IPV4_SRC_ADDR \ + | SESSION_ATTR_L4_DST_PORT \ + | SESSION_ATTR_IPV4_DST_ADDR \ + | SESSION_ATTR_IF_NAME) + +struct dp_session_info { + enum dp_session_attr query; + uint64_t se_id; + uint16_t se_flags; + uint8_t se_protocol; + uint8_t se_protocol_state; + uint64_t se_pkts_in; + uint64_t se_bytes_in; + uint64_t se_create_time; /* time session was created */ + uint64_t se_pkts_out; + uint64_t se_bytes_out; + + // address + int se_af; + uint16_t se_src_port; + uint32_t se_src_addr; + uint16_t se_dst_port; + uint32_t se_dst_addr; + const char *se_ifname; + const char *se_app_name; + const char *se_app_proto; + const char *se_app_type; + + // firewall + const char *se_fwd_status; + + // misc + time_t timestamp; + uint64_t duration; /* seconds */ +}; + #define SESSION_STATE_FIRST SESSION_STATE_NONE #define SESSION_STATE_LAST SESSION_STATE_CLOSED #define SESSION_STATE_SIZE (SESSION_STATE_LAST + 1) @@ -233,6 +290,12 @@ void *dp_session_get_private(int id, const struct session *session); int dp_session_table_walk(dp_session_walk_t *fn, void *data, unsigned int types); +/** + * Query a session's info. + */ +int dp_session_query(struct session *s, enum dp_session_attr query, + struct dp_session_info *info); + /** * Get a session's unique id. * diff --git a/meson.build b/meson.build index a562dd9e..8a6515c5 100644 --- a/meson.build +++ b/meson.build @@ -80,6 +80,7 @@ subdir('src') if get_option('with_tests').enabled() subdir('src/pipeline/nodes/sample') + subdir('src/pipeline/nodes/flowstat') subdir('tests/whole_dp') endif diff --git a/scripts/vyatta-generate-pb-perl.pl b/scripts/vyatta-generate-pb-perl.pl index d88d9349..12e3c1e6 100755 --- a/scripts/vyatta-generate-pb-perl.pl +++ b/scripts/vyatta-generate-pb-perl.pl @@ -8,7 +8,10 @@ use MIME::Base64; use Google::ProtocolBuffers; -my ($first, $proto_dir, $proto_file) = split(/\//, $ARGV[0]); +my @arg0 = split(/\//, $ARGV[0]); +my $proto_dir = $arg0[$#arg0 - 1]; +my $proto_file = $arg0[$#arg0]; + my $target_dir = $ARGV[1]; if (substr($proto_file, -6) eq ".proto") { diff --git a/src/npf/dpi/dpi.c b/src/npf/dpi/dpi.c index cd97de9b..49ddce1f 100644 --- a/src/npf/dpi/dpi.c +++ b/src/npf/dpi/dpi.c @@ -523,6 +523,22 @@ dpi_app_type_name_to_id(uint8_t engine_id, const char *type_name) return engine ? engine->type_to_id(type_name) : DPI_APP_ERROR; } +const char* +dpi_app_id_to_name(uint8_t engine_id, uint32_t app) +{ + struct dpi_engine_procs *engine = NULL_ENGINE; + ENGINE_PROC_FIND(engine, engine_id, appid_to_name); + return engine ? engine->appid_to_name(app) : NULL; +} + +const char* +dpi_app_type_to_name(uint8_t engine_id, uint32_t type) +{ + struct dpi_engine_procs *engine = NULL_ENGINE; + ENGINE_PROC_FIND(engine, engine_id, apptype_to_name); + return engine ? engine->apptype_to_name(type) : NULL; +} + void dpi_info_json(struct dpi_flow *dpi_flow, json_writer_t *json) { diff --git a/src/npf/dpi/dpi_internal.h b/src/npf/dpi/dpi_internal.h index db87f379..2fc259e6 100644 --- a/src/npf/dpi/dpi_internal.h +++ b/src/npf/dpi/dpi_internal.h @@ -404,8 +404,21 @@ struct dpi_engine_procs { */ size_t (*info_log)(struct dpi_engine_flow *flow, char *buf, size_t buf_len); + + /** + * Get name of app id. + */ + const char* (*appid_to_name)(uint32_t app); + + /** + * Get type of type id. + */ + const char* (*apptype_to_name)(uint32_t type); }; +const char *dpi_app_id_to_name(uint8_t engine_id, uint32_t app); +const char *dpi_app_type_to_name(uint8_t engine_id, uint32_t type); + bool no_app_id(uint32_t app_id); bool no_app_type(uint32_t app_type); diff --git a/src/npf/dpi/dpi_user.c b/src/npf/dpi/dpi_user.c index 8fde1f0f..cfa55125 100644 --- a/src/npf/dpi/dpi_user.c +++ b/src/npf/dpi/dpi_user.c @@ -317,4 +317,6 @@ struct dpi_engine_procs user_engine_procs = { .type_to_id = dpi_user_type_to_id, .info_json = dpi_user_flow_json, .info_log = dpi_user_flow_log, + .appid_to_name = dpi_user_id_to_name, + .apptype_to_name = dpi_user_type_to_name, }; diff --git a/src/npf/dpi/ndpi.c b/src/npf/dpi/ndpi.c index 4158ff9d..c423012f 100644 --- a/src/npf/dpi/ndpi.c +++ b/src/npf/dpi/ndpi.c @@ -537,4 +537,6 @@ struct dpi_engine_procs ndpi_engine_procs = { .type_to_id = dpi_ndpi_app_type_name_to_id, .info_json = dpi_ndpi_info_json, .info_log = dpi_ndpi_info_log, + .appid_to_name = dpi_ndpi_app_id_to_name, + .apptype_to_name = dpi_ndpi_app_type_to_name, }; diff --git a/src/npf/npf_dataplane_session.c b/src/npf/npf_dataplane_session.c index 7379e7b5..9ec57773 100644 --- a/src/npf/npf_dataplane_session.c +++ b/src/npf/npf_dataplane_session.c @@ -113,6 +113,12 @@ static int dps_feature_nat_info(void *data, uint32_t *taddr, uint16_t *tport) return npf_session_feature_nat_info(se, taddr, tport); } +static void dps_feature_query(struct dp_session_info *info, struct session *s, + struct session_feature *sf) +{ + return npf_session_feature_query(info, s, sf); +} + /* Callbacks for the npf_session_t */ static const struct session_feature_ops ops = { .expired = dps_feature_expire, @@ -120,6 +126,7 @@ static const struct session_feature_ops ops = { .json = dps_feature_json, .log = dps_feature_log, .nat_info = dps_feature_nat_info, + .query = dps_feature_query, }; /* diff --git a/src/npf/npf_session.c b/src/npf/npf_session.c index a220bc43..07e68474 100644 --- a/src/npf/npf_session.c +++ b/src/npf/npf_session.c @@ -2089,3 +2089,28 @@ int npf_session_npf_pack_activate(struct npf_session *se, struct ifnet *ifp) se->s_flags |= SE_ACTIVE; return 0; } + +static int get_dpi_info(uint8_t engine, uint32_t app, uint32_t proto, + uint32_t type, void *data) +{ + if (no_app_id(app) && no_app_id(proto) && no_app_type(type)) + return 0; + + struct dp_session_info *p = data; + p->se_app_name = dpi_app_id_to_name(engine, app); + p->se_app_proto = dpi_app_id_to_name(engine, proto); + p->se_app_type = dpi_app_type_to_name(engine, type); + return 1; +} + +void npf_session_feature_query(struct dp_session_info *info, + struct session *s __unused, + struct session_feature *sf) +{ + npf_session_t *se = sf->sf_data; + enum dp_session_attr query = info->query; + + /* DPI query */ + if (query & SESSION_ATTR_DPI && se->s_dpi) + dpi_flow_for_each_engine(se->s_dpi, get_dpi_info, info); +} diff --git a/src/npf/npf_session.h b/src/npf/npf_session.h index cec0a81e..158cc7c7 100644 --- a/src/npf/npf_session.h +++ b/src/npf/npf_session.h @@ -135,6 +135,8 @@ npf_nat_t *npf_session_retnat(npf_session_t *se, const int di, bool *forw); void npf_session_feature_json(json_writer_t *json, npf_session_t *se); void npf_session_feature_log(enum session_log_event event, struct session *s, struct session_feature *sf); +void npf_session_feature_query(struct dp_session_info *info, struct session *s, + struct session_feature *sf); int npf_session_feature_nat_info(npf_session_t *se, uint32_t *taddr, uint16_t *tport); diff --git a/src/pipeline/nodes/flowstat/meson.build b/src/pipeline/nodes/flowstat/meson.build new file mode 100644 index 00000000..b751fe62 --- /dev/null +++ b/src/pipeline/nodes/flowstat/meson.build @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: LGPL-2.1-only +# Copyright (c) 2021, SafePoint. All rights reserved. +# Copyright (c) 2020, AT&T Intellectual Property. All rights reserved. + +flowstat_plugin_sources = files( + 'src/flowstat.c' +) + +subdir('protobuf') + +flowstat_plugin = shared_module('flowstat_plugin', + sources: [flowstat_plugin_sources, flowstat_generated_protobuf_c], + dependencies: [dl_dep, dpdk_dep, proto_c_dep, threads_dep, urcu_cds_dep, urcu_dep, urcu_qsbr_dep], + include_directories: public_include, + name_prefix: '', + install: true, + install_dir: get_option('prefix') / get_option('libdir') / meson.project_name() / 'pipeline' / 'plugins' +) + +flowstat_include = include_directories('src') diff --git a/src/pipeline/nodes/flowstat/protobuf/FlowStatFeatConfig.proto b/src/pipeline/nodes/flowstat/protobuf/FlowStatFeatConfig.proto new file mode 100644 index 00000000..36ec3bdd --- /dev/null +++ b/src/pipeline/nodes/flowstat/protobuf/FlowStatFeatConfig.proto @@ -0,0 +1,12 @@ +// Copyright (c) 2021, SafePoint. All rights reserved. +// Copyright (c) 2018-2019, AT&T Intellectual Property. All rights reserved. +// +// SPDX-License-Identifier: LGPL-2.1-only +// + +syntax="proto2"; + +message FlowStatFeatConfig { + optional bool is_active = 1; + optional string if_name = 2; +} \ No newline at end of file diff --git a/src/pipeline/nodes/flowstat/protobuf/FlowStatFeatOp.proto b/src/pipeline/nodes/flowstat/protobuf/FlowStatFeatOp.proto new file mode 100644 index 00000000..2e6f79c4 --- /dev/null +++ b/src/pipeline/nodes/flowstat/protobuf/FlowStatFeatOp.proto @@ -0,0 +1,15 @@ +// Copyright (c) 2021, SafePoint. All rights reserved. +// Copyright (c) 2018-2019, AT&T Intellectual Property. All rights reserved. +// +// SPDX-License-Identifier: LGPL-2.1-only +// + +syntax="proto2"; + +message FlowStatOpReq { + //empty +} + +message FlowStatOpResp { + optional uint32 count = 1; +} \ No newline at end of file diff --git a/src/pipeline/nodes/flowstat/protobuf/meson.build b/src/pipeline/nodes/flowstat/protobuf/meson.build new file mode 100644 index 00000000..57a1f91f --- /dev/null +++ b/src/pipeline/nodes/flowstat/protobuf/meson.build @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: LGPL-2.1-only +# Copyright (c) 2021, SafePoint. All rights reserved. +# Copyright (c) 2020, AT&T Intellectual Property. All rights reserved. + + +flowstat_protobuf_sources = [ + 'FlowStatFeatConfig.proto', + 'FlowStatFeatOp.proto', +] + +install_data(flowstat_protobuf_sources, + install_dir: get_option('datadir') / 'flowstat_plugin' / 'protobuf' +) + +flowstat_generated_protobuf_c = [] +flowstat_generated_protobuf_c_headers = [] +foreach protobuf_definition : flowstat_protobuf_sources + generated_c = custom_target('c_' + protobuf_definition, + command: [protoc, '--proto_path=@CURRENT_SOURCE_DIR@', '--c_out=@OUTDIR@', '@INPUT@'], + input: protobuf_definition, + output: ['@BASENAME@.pb-c.c', '@BASENAME@.pb-c.h'], + ) + flowstat_generated_protobuf_c += generated_c + flowstat_generated_protobuf_c_headers += generated_c[1] + + protobuf_generated_perl += custom_target('pl_' + protobuf_definition, + command: [perl_protobuf_generator, '@INPUT@', '@OUTDIR@', '@CURRENT_SOURCE_DIR@'], + input: protobuf_definition, + output: '@BASENAME@.pm', + install: true, + install_dir: 'share/perl5/flowstat_plugin/proto' + ) +endforeach \ No newline at end of file diff --git a/src/pipeline/nodes/flowstat/scripts/flowstat_get_state.py b/src/pipeline/nodes/flowstat/scripts/flowstat_get_state.py new file mode 100755 index 00000000..75802e18 --- /dev/null +++ b/src/pipeline/nodes/flowstat/scripts/flowstat_get_state.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, SafePoint. All rights reserved. +# +# SPDX-License-Identifier: LGPL-2.1-only +# + +import argparse +import json +import vplaned + +from datetime import datetime +from flowstatd import FlowStatDb + +DB_PATH = '/var/log/flowstat.db' + +PERIOD_MAP = { + 'now': 0, + '5m': 300, + '1h': 3600, + '1d': 86400, +} + +SESSION_STATE_TERMINATING = 3 + + +def _parse_sessions(data): + ss = data['config']['sessions'] + + result = [] + for sid, s in ss.items(): + features = s['features'] or [{}] + dpi_engines = features[0].get('dpi', {}).get('engines', []) + dpi = dpi_engines[0] if dpi_engines else {} + if dpi.get('gen_state') == SESSION_STATE_TERMINATING: + continue + d = { + 'timestamp': None, + 'session_id': sid, + 'src_addr': s['src_addr'], + 'src_port': s['src_port'], + 'dst_addr': s['dst_addr'], + 'dst_port': s['dst_port'], + 'in_bytes': s['counters']['bytes_in'], + 'in_pkts': s['counters']['packets_in'], + 'protocol': s['proto'], + 'out_bytes': s['counters']['bytes_out'], + 'out_pkts': s['counters']['packets_out'], + 'if_name': s['interface'], + 'app_name': dpi.get('app-name', ''), + 'app_proto': dpi.get('proto-name', ''), + 'app_type': dpi.get('type', ''), + } + result.append(d) + + return result + + +def _get_sessions(): + cmd = "session-op show sessions full" + + with vplaned.Controller() as controller: + for dp in controller.get_dataplanes(): + with dp: + rv = dp.json_command(cmd) + rv = _parse_sessions(rv) + return rv + + +def _output(items): + """Make sort by yang.""" + res = { + 'items': items + } + + print(json.dumps(res)) + + +def get_top(key, sort_by, start_time, end_time, if_name=None, limit=10): + if start_time == 'now': + db = FlowStatDb(':memory:') + db.connect() + data = _get_sessions() + db.prepare_temp_data(data) + else: + db = FlowStatDb(DB_PATH) + db.connect() + + res = db.query_top(key, sort_by, start_time, end_time, if_name=if_name, limit=limit) + _output(res) + db.close() + + +def get_timeseries(key, start_time, end_time, if_name=None): + db = FlowStatDb(DB_PATH) + db.connect() + res = db.query_time_series(key, start_time, end_time, if_name=if_name) + _output(res) + db.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Flowstat query') + parser.add_argument('--intf', help='Interface') + parser.add_argument('--type', help='Query type', + choices=['top', 'timeseries'], required=True) + parser.add_argument('--key', help='Key', required=True) + parser.add_argument('--unit', help='Unit') + parser.add_argument('--period', help='Period', + choices=PERIOD_MAP.keys(), required=True) + args = parser.parse_args() + + # get start time, end time + if args.period == 'now': + start = end = 'now' + + if args.type == 'timeseries': + _output([]) + exit(0) + else: + end = int(datetime.utcnow().timestamp()) + start = end - PERIOD_MAP[args.period] + + # fix ALL interfaces + if args.intf == 'ALL': + args.intf = None + + if args.type == 'top': + get_top(args.key, args.unit, start, end, if_name=args.intf) + exit(0) + + if args.type == 'timeseries': + get_timeseries(args.key, start, end, if_name=args.intf) + exit(0) diff --git a/src/pipeline/nodes/flowstat/scripts/flowstat_update_logrotate b/src/pipeline/nodes/flowstat/scripts/flowstat_update_logrotate new file mode 100755 index 00000000..6749bfcc --- /dev/null +++ b/src/pipeline/nodes/flowstat/scripts/flowstat_update_logrotate @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, SafePoint. +# Copyright (c) 2019, AT&T Intellectual Property. +# All rights reserved. +# +# SPDX-License-Identifier: GPL-2.0-only +# + +"""Script to configure logrotate of flowstat""" + +from vyatta import configd +from vplaned import Controller + +LOG_FILE = "/var/log/flowstat.log" +LOG_CONF = "/etc/logrotate.d/flowstat" + + +def main(): + try: + client = configd.Client() + except Exception as exc: + print("Cannot establish client session: '{}'".format(str(exc).strip())) + return 1 + + path = "service flow-stat logrotate" + + try: + cfg = client.tree_get_dict(path)['logrotate'] + except configd.Exception: + return 0 + + files_val = cfg.get('files') + size_val = cfg.get('size') + maxage_val = cfg.get('maxage') + + with open(LOG_CONF, 'w') as f: + data = '''\ +%s { + hourly + missingok + rotate %d + size=%dk + maxage %d + nocompress +} +''' % (LOG_FILE, files_val, size_val, maxage_val) + f.write(data) + + return 0 + + +if __name__ == '__main__': + main() diff --git a/src/pipeline/nodes/flowstat/scripts/flowstatd b/src/pipeline/nodes/flowstat/scripts/flowstatd new file mode 100755 index 00000000..67ef2203 --- /dev/null +++ b/src/pipeline/nodes/flowstat/scripts/flowstatd @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, SafePoint. All rights reserved. +# +# SPDX-License-Identifier: LGPL-2.1-only +# + +from flowstatd import run_collect + + +run_collect() diff --git a/src/pipeline/nodes/flowstat/scripts/flowstatd.py b/src/pipeline/nodes/flowstat/scripts/flowstatd.py new file mode 100755 index 00000000..ef85fb7e --- /dev/null +++ b/src/pipeline/nodes/flowstat/scripts/flowstatd.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, SafePoint. All rights reserved. +# +# SPDX-License-Identifier: LGPL-2.1-only +# + +import glob +import json +import os +import sqlite3 +import time +import traceback +from datetime import datetime + +# Check sqlite3 version +if sqlite3.sqlite_version_info[0] * 1000 + sqlite3.sqlite_version_info[1] <= 3024: + raise RuntimeError('sqlite3>=3.24.0 required') + +DB_FILE = '/var/log/flowstat.db' +LOG_DIR = '/var/log' +LOG_FILE = f'{LOG_DIR}/flowstat.log' + +QUERY_PERIOD_MAP = { + 300: (5, 10), # last 5 minute, period 5, group by 10s + 3600: (60, 60), # last 1 hour, period 60, group by 1m + 86400: (3600, 3600), # last 1 day, period 3600, group by 1h +} + +PERIOD_LIST = sorted(i[0] for i in QUERY_PERIOD_MAP.values()) + +AGG_TABLE_LIST = [ + {'name': 'grp_flow_src', 'alias': 'src', 'key': 'src_addr'}, + {'name': 'grp_flow_dst', 'alias': 'dst', 'key': 'dst_addr'}, + {'name': 'grp_app_name', 'alias': 'app', 'key': 'app_name'}, + {'name': 'grp_app_proto', 'alias': 'app_proto', 'key': 'app_proto'}, + {'name': 'grp_app_type', 'alias': 'app_type', 'key': 'app_type'}, +] + +AGG_COMMON_FIELDS = [ + {'name': 'in_bytes', 'op': 'sum'}, + {'name': 'out_bytes', 'op': 'sum'}, +] + +AGG_TABLE_TPL = ''' +{name}_{period} ( + timestamp interger, + {key} char(32), + in_bytes integer, + out_bytes integer, + if_name char(32), + primary key (if_name, timestamp, {key}) +) +''' + +METADATA_TABLE_TPL = ''' +metadata ( + id integer primary key autoincrement, + data text +) +''' + +TEMPFLOW_TABLE_COLS = ''' + timestamp integer, + session_id integer, + src_addr char(16), + src_port integer, + dst_addr char(16), + dst_port integer, + in_bytes integer, + in_pkts integer, + protocol integer, + out_bytes integer, + out_pkts integer, + app_name char(32), + app_proto char(32), + app_type char(32), + if_name char(32) +''' + +TEMPFLOW_TABLE_TPL = f''' +tempflow ( + id integer primary key autoincrement, + {TEMPFLOW_TABLE_COLS} +) +''' + + +class FlowStatDb: + def __init__(self, dbpath, read_limit=10000, insert_batch=1000): + self.dbpath = dbpath + self.temp_read_limit = read_limit + self.temp_insert_batch = insert_batch + self.metadata = { + 'last_log': { + 'index': -1, + 'position': 0, + 'timestamp': 0, + }, + } + self.conn = None + + @property + def last_log(self): + return self.metadata['last_log'] + + def connect(self): + conn = sqlite3.connect(self.dbpath) + # print("Opened database %s successfully" % file_path) + + sql = '' + + # setup tables + for period in PERIOD_LIST: + for table in AGG_TABLE_LIST: + sql += (f''' + create table if not exists {AGG_TABLE_TPL}; + ''').format(**table, period=period) + + # meta data + sql += f''' + create table if not exists {METADATA_TABLE_TPL}; + ''' + + # temp flow + sql += f''' + create temp table if not exists {TEMPFLOW_TABLE_TPL}; + ''' + + conn.executescript(sql) + + self.conn = conn + + def close(self): + self.conn.close() + + def commit(self): + self.conn.commit() + + def _upsert(self, table, primary_keys, cols, values, cur=None): + c = cur or self.conn.cursor() + conflict_cols = ','.join(primary_keys) + upsert = ','.join(f'{i}=excluded.{i}' for i in cols if i not in primary_keys) + values_wc = ','.join('?' * len(cols)) + cols = ','.join(cols) + sql = f''' + insert into {table} ({cols}) values ({values_wc}) + on conflict ({conflict_cols}) + do update set {upsert} + ''' + # print(sql) + c.executemany(sql, values) + + if cur is None: + c.close() + + def _read_logfile(self, logfile, index, limit, period, last_log): + line_number = 0 + last_ts = None + need_reading = True + changed_ts = False + while need_reading and index >= 0: + if index == 0: + curfile = logfile + else: + curfile = '%s.%d' % (logfile, index) + + if not os.path.isfile(curfile): + index -= 1 + continue + + with open(curfile, 'r') as f: + # seek first file only + if index == last_log['index']: + f.seek(last_log['position']) + + while True: + prev_pos = f.tell() + line = f.readline() + line_number += 1 + + if line == '': + # EOL, go to next file + index -= 1 + break + + # parse log + d = [i.split('=') for i in line.strip().split(' ')] + d = {i[0]: (i[1] if len(i) == 2 else None) for i in d} + + # fix string val + for k in ['if_name']: + v = d.get(k) + if v: + d[k] = v[1:len(v) - 1] + + # save last position + ts = int(d['timestamp']) + ts = ts - (ts % period) + if last_ts is None: + last_ts = ts + # save timestamp for first line + last_log['timestamp'] = int(d['timestamp']) + + # jump to new timestamp period + if ts > last_ts: + last_ts = ts + last_log['index'] = index + last_log['position'] = prev_pos + last_log['timestamp'] = int(d['timestamp']) + + yield d + + # only break if move over one period + if line_number >= limit: + if changed_ts: + print(f'Read limited line:{line_number}') + need_reading = False + break + elif line_number % limit == 0: + time.sleep(1) + + def prepare_temp_data(self, data): + c = self.conn.cursor() + + cols = [i.split()[0] for i in TEMPFLOW_TABLE_COLS.split(',')] + values_wc = ','.join('?' * len(cols)) + cols_name = ','.join(cols) + + to_inserts = [] + for d in data: + to_inserts.append(d) + + # batch insert + if len(to_inserts) >= self.temp_insert_batch: + sql = f''' + insert into tempflow + ({cols_name}) + values ({values_wc}) + ''' + data_insert = [[d[i] for i in cols] for d in to_inserts] + c.executemany(sql, data_insert) + # print('Inserted %d items' % len(to_inserts)) + to_inserts = [] + + # remain + if len(to_inserts): + sql = f''' + insert into tempflow + ({cols_name}) + values ({values_wc}) + ''' + data_insert = [[d[i] for i in cols] for d in to_inserts] + c.executemany(sql, data_insert) + # print('Inserted %d items' % len(to_inserts)) + + c.close() + + def _compute_statistics(self): + c = self.conn.cursor() + + for table in AGG_TABLE_LIST: + for period in PERIOD_LIST: + tbname = f'{table["name"]}_{period}' + + # find last time of package + sql = f''' + select timestamp + from {tbname} + order by timestamp desc + limit 1; + ''' + c.execute(sql) + last_time = c.fetchone() + if last_time: + where = f'timestamp >= {last_time[0]}' + else: + where = '1=1' + + # each period depends on prev periods + index = PERIOD_LIST.index(period) + if index == 0: + prev_table = 'tempflow' + else: + prev_period = PERIOD_LIST[index - 1] + prev_table = f'{table["name"]}_{prev_period}' + + select = ','.join(f'{i["op"]}({i["name"]})' if 'op' in i else i['name'] + for i in AGG_COMMON_FIELDS) + sql = f''' + select if_name, (timestamp - (timestamp % {period})) ts, + {table['key']}, + {select} + from {prev_table} + where {where} + group by if_name,ts,{table['key']} + ''' + + # print(sql) + c.execute(sql) + records = c.fetchall() + # print(records) + # print(f'Found new {tbname} {len(records)} items') + + # insert into table + primary_keys = ['if_name', 'timestamp', table['key']] + extra_cols = [i['name'] for i in AGG_COMMON_FIELDS] + cols = primary_keys + extra_cols + self._upsert(tbname, primary_keys, cols, records, cur=c) + print(f'Updated {tbname} {c.rowcount} items') + c.close() + + def _check_logfile(self, logfile, index, position, timestamp): + # find prev file by index + if index == 0: + curfile = logfile + else: + curfile = '%s.%d' % (logfile, index) + + try: + with open(curfile, 'r') as f: + f.seek(position) + line = f.readline() + ts = 'timestamp=%d' % timestamp + return ts in line + except Exception: + return False + + def _find_last_logfile(self, logfile, last_log): + default = { + 'index': 0, + 'position': 0, + 'timestamp': 0, + } + + if last_log['index'] == -1: + # no prev info found, load from current file + return default + + # check last index + if self._check_logfile(logfile, last_log['index'], + last_log['position'], + last_log['timestamp']): + return last_log + else: + # last file not valid, check all log + files = glob.glob(f'{logfile}*') + for file in sorted(files): + # extract index from file name + if file == logfile: + index = 0 + else: + index = int(file.rsplit('.', 1)[-1]) # log.1,... + + if self._check_logfile(logfile, index, last_log['position'], + last_log['timestamp']): + last_log['index'] = index + return last_log + else: + # nothing found + return default + + def find_last_logfile(self, logfile): + last_log = self._find_last_logfile(logfile, self.last_log) + self.last_log.update(last_log) + + def _load_metadata(self): + c = self.conn.cursor() + sql = f''' + select data + from metadata + ''' + c.execute(sql) + rec = c.fetchone() + print(f'read lastlog: {rec}') + if rec: + data = json.loads(rec[0]) + self.metadata.update(data) + c.close() + + def _save_metadata(self): + print(f'save lastlog: {self.last_log}') + c = self.conn.cursor() + upsert_cols = [ + 'id', + 'data', + ] + value = [ + 1, + json.dumps(self.metadata), + ] + self._upsert('metadata', ['id'], upsert_cols, [value], cur=c) + + c.close() + + def collect_from_logfile(self, logfile): + self._load_metadata() + self.find_last_logfile(logfile) + data = self._read_logfile(logfile, self.last_log['index'], + self.temp_read_limit, PERIOD_LIST[0], self.last_log) + self.prepare_temp_data(data) + self._compute_statistics() + self._save_metadata() + + def _guess_period(self, start_time, end_time): + diff = end_time - start_time + for k, v in QUERY_PERIOD_MAP.items(): + if diff <= k: + return v + return 3600, 3600 # default + + def _format_as_json(self, result, fields): + result = [dict(zip(fields, i)) for i in result] + return result + + def _find_table(self, alias): + table = (i for i in AGG_TABLE_LIST if i['alias'] == alias) + table = next(table, None) + if not table: + raise RuntimeError(f'Table not found for alias:{alias}') + return table + + def query_top(self, alias, attr, start_time, end_time, if_name=None, limit=10): + table = self._find_table(alias) + + if start_time == 'now': + table_name = 'tempflow' + where = '1=1' + else: + period, subperiod = self._guess_period(start_time, end_time) + table_name = f'{table["name"]}_{period}' + where = f'timestamp between {start_time} and {end_time}' + + if if_name: + where += f' and if_name = "{if_name}"' + + c = self.conn.cursor() + sql = f''' + select {table['key']}, + sum(in_bytes) in_bytes, + sum(out_bytes) out_bytes, + sum(in_bytes + out_bytes) bytes + from {table_name} + where {where} + group by {table['key']} + order by {attr} desc + limit {limit} + ''' + # print(sql) + c.execute(sql) + res = c.fetchall() + c.close() + res = self._format_as_json(res, ['key', 'in_bytes', 'out_bytes', 'bytes']) + return res + + def query_time_series(self, alias, start_time, end_time, if_name=None): + period, subperiod = self._guess_period(start_time, end_time) + table = self._find_table(alias) + c = self.conn.cursor() + + where = f'timestamp between {start_time} and {end_time}' + if if_name: + where += f' and if_name = "{if_name}"' + + sql = f''' + select timestamp-(timestamp%{subperiod}) ts, + sum(in_bytes), + sum(out_bytes), + sum(in_bytes + out_bytes) + from {table['name']}_{period} + where {where} + group by ts + order by timestamp asc + ''' + # print(sql) + c.execute(sql) + res = c.fetchall() + c.close() + res = self._format_as_json(res, ['timestamp', 'in_bytes', 'out_bytes', 'bytes']) + return res + + def vacuum(self): + c = self.conn.cursor() + m = {v[0]: k for k, v in QUERY_PERIOD_MAP.items()} + for table in AGG_TABLE_LIST: + for period in PERIOD_LIST: + time_diff = m[period] + tsnow = int(datetime.utcnow().timestamp()) + start = tsnow - (tsnow % period) - time_diff * 2 + + # Period is list depend, sub period can not delete item if parent + # still need them. + next_index = PERIOD_LIST.index(period) + 1 + if next_index < len(PERIOD_LIST): + next_period = PERIOD_LIST[next_index] + sql = f''' + select timestamp from {table['name']}_{next_period} + order by timestamp desc + limit 1 + ''' + c.execute(sql) + res = c.fetchone() + if res: + start = min(start, res[0]) + + # delete + sql = f''' + delete from {table['name']}_{period} + where timestamp < {start} + ''' + # print(sql) + c.execute(sql) + if c.rowcount: + print(f'Deleted from {table["name"]}_{period} {c.rowcount} items') + + c.close() + + def show_info(self): + c = self.conn.cursor() + + for table in AGG_TABLE_LIST: + for period in PERIOD_LIST: + table_name = f'{table["name"]}_{period}' + sql = f''' + select timestamp + from {table_name} + order by timestamp asc + limit 1 + ''' + c.execute(sql) + start = c.fetchone() + if start: + start = start[0] + else: + start = '-' + + sql = f''' + select timestamp + from {table_name} + order by timestamp desc + limit 1 + ''' + c.execute(sql) + end = c.fetchone() + if end: + end = end[0] + else: + end = '-' + + sql = f'select count(*) from {table["name"]}_{period}' + c.execute(sql) + count = c.fetchone() + if count: + count = count[0] + else: + count = 0 + + print(f'{table_name:<20} ({count:08}) {start:>10} => {end:>10}') + + c.close() + + +def collect_statistics(db_file, log_file): + st = time.time() + + db = FlowStatDb(db_file) + db.connect() + db.collect_from_logfile(log_file) + db.vacuum() + # db.show_info() + db.commit() + db.close() + + print('Collect finished in %.2fs' % (time.time() - st)) + + +def run_collect(): + while True: + try: + collect_statistics(DB_FILE, LOG_FILE) + except Exception as e: + traceback.print_exc() + time.sleep(10) + + +if __name__ == '__main__': + run_collect() diff --git a/src/pipeline/nodes/flowstat/scripts/vplane-flowstat.pl b/src/pipeline/nodes/flowstat/scripts/vplane-flowstat.pl new file mode 100755 index 00000000..f393c577 --- /dev/null +++ b/src/pipeline/nodes/flowstat/scripts/vplane-flowstat.pl @@ -0,0 +1,72 @@ +#!/usr/bin/perl +# Module: vplane-flowstat.pl +# +# **** License **** +# +# Copyright (c) 2021, SafePoint. +# Copyright (c) 2019, AT&T Intellectual Property. +# Copyright (c) 2015-2016 by Brocade Communications Systems, Inc. +# All rights reserved. +# +# SPDX-License-Identifier: GPL-2.0-only +# +# **** End License **** + + +use strict; +use warnings; +use lib '/opt/vyatta/share/perl5'; +use Getopt::Long; +use Vyatta::Config; +use Vyatta::VPlaned; + +use vyatta::proto::FlowStatFeatConfig; + +# +# main +# +my ( $cmd, $intf, $enable ) = ("", "", ""); + +GetOptions( + "cmd=s" => \$cmd, + "intf=s" => \$intf, + "enable=s" => \$enable, +); + +sub config_dataplane_flowstat { + my $cstore = new Vyatta::VPlaned; + + my $is_active = $enable eq 'true' ? 1 : 0; + + my $config = FlowStatFeatConfig->new({ + is_active => $is_active, + if_name => $intf + }); + + $cstore->store_pb( + "interface dataplane flowstat $intf enable", + $config, + "fstat:fstat-feat"); +} + +sub config_global_flowstat { + my $cstore = new Vyatta::VPlaned; + + my $is_active = $enable eq 'true' ? 1 : 0; + + my $config = FlowStatFeatConfig->new({ + is_active => $is_active, + if_name => "" + }); + + $cstore->store_pb( + "service flowstat disable", + $config, + "fstat:fstat-feat"); +} + +if ($cmd eq "cfg_global") { + config_global_flowstat(); +} else { + config_dataplane_flowstat(); +} diff --git a/src/pipeline/nodes/flowstat/src/flowstat.c b/src/pipeline/nodes/flowstat/src/flowstat.c new file mode 100644 index 00000000..724b1795 --- /dev/null +++ b/src/pipeline/nodes/flowstat/src/flowstat.c @@ -0,0 +1,678 @@ +/* + * Flow stat pipeline feature node + * + * Copyright (c) 2021, SafePoint. All rights reserved. + * Copyright (c) 2017-2021, AT&T Intellectual Property. All rights reserved. + * Copyright (c) 2017 by Brocade Communications Systems, Inc. + * All rights reserved. + * + * SPDX-License-Identifier: LGPL-2.1-only + * + * To generate the protobuf message source: + * "protoc-c -I=. --c_out=. ./FlowStatFeatConfig.proto" + * + * To compile flow_stat as a standalone: + * "gcc -shared -fPIC flowstat.c -I/usr/include/vyatta-dataplane + * $(pkg-config --cflags libdpdk) -o libflowstat.so" + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* RCU flavor */ +#include /* RCU Lock-free hash table */ +#include +#include +#include +#include +#include + +#include "compiler.h" +#include "debug.h" +#include "dp_session.h" +#include "feature_commands.h" +#include "feature_plugin.h" +#include "json_writer.h" +#include "pipeline.h" + +#include "flowstat.h" +#include "FlowStatFeatConfig.pb-c.h" +#include "FlowStatFeatOp.pb-c.h" + +#define RTE_LOGTYPE_FLOWSTAT RTE_LOGTYPE_USER4 + +enum fstat_dispositions { FLOW_STAT_ACCEPT, FLOW_STAT_NUM }; + +enum log_level { + NOT_SET_LEVEL = 0, + CRIT_LEVEL = 10, + WARN_LEVEL = 20, + INFO_LEVEL = 30, + DEBUG_LEVEL = 40, +}; + +struct session_private { + bool locked; + bool is_long_lived; /* stat calculated by delta */ + time_t last_seen; + time_t last_seen_tw; + uint64_t last_pkts_in; + uint64_t last_bytes_in; + uint64_t last_pkts_out; + uint64_t last_bytes_out; + + /* cached DPI */ + const char *app; + const char *app_proto; + const char *app_type; +}; + +/* Extra info for session */ +#define SESSION_PRIVATE_ID 10001000 +#define LOG_BUFFER_LIMIT 10000 +#define LOG_MSG_SIZE 512 +#define EXPORTER_FILE_BUFSIZ (LOG_MSG_SIZE * 1000) +#define EXPORTER_INTERVAL 5 +#define LOG_TW_SESSION_INTERVAL 1 +#define NPF_TCPS_TIME_WAIT 11 + +/* Buffer log */ +static uint32_t log_buffer_count; + +/* Config */ +static bool is_enabled_global = true; +static int debug_level = INFO_LEVEL; + +/* + * Nodes populated into the queue. + */ +struct lognode { + struct dp_session_info *info; /* Node content */ + struct cds_wfcq_node node; /* Chaining in queue */ +}; + +struct cds_wfcq_head logqueue_head; /* Queue head */ +struct cds_wfcq_tail logqueue_tail; /* Queue tail */ + +/* + * Enabled interfaces hash table. + */ +struct intf_node { + char name[IFNAMSIZ]; /* Interface name */ + struct cds_lfht_node node; /* Chaining in hash table */ +}; + +struct cds_lfht *enabled_intf_ht; + +#define ENABLED_INTF_HASH_MIN 4 +#define ENABLED_INTF_HASH_MAX 0 /* Unlimited */ + +static inline uint32_t intf_hash_name(const char *ifname) +{ + char __ifname[IFNAMSIZ] __rte_aligned(sizeof(uint32_t)); + int len = MIN(strlen(ifname), sizeof(__ifname)); + + memcpy(__ifname, ifname, len); + return rte_jhash(__ifname, len, 0); +} + +static inline int intf_match(struct cds_lfht_node *ht_node, const void *arg) +{ + const struct intf_node *node; + const char *key = arg; + + node = caa_container_of(ht_node, struct intf_node, node); + if ((strncmp(key, node->name, IFNAMSIZ) == 0)) + return 1; + + return 0; +} + +static struct intf_node *intf_lookup(const char *ifname) +{ + struct intf_node *node = NULL; + struct cds_lfht_iter iter; + struct cds_lfht_node *ht_node; + + cds_lfht_lookup(enabled_intf_ht, intf_hash_name(ifname), intf_match, + ifname, &iter); + + ht_node = cds_lfht_iter_get_node(&iter); + if (ht_node) + node = caa_container_of(ht_node, struct intf_node, node); + + return node; +} + +static void format_session_log(char *buf, struct dp_session_info *info) +{ + time_t utc_now = mktime(gmtime(&info->timestamp)); + struct tm *ts = localtime(&info->timestamp); + + char srcip_str[INET6_ADDRSTRLEN]; + char dstip_str[INET6_ADDRSTRLEN]; + + inet_ntop(info->se_af, &info->se_src_addr, srcip_str, + sizeof(srcip_str)); + inet_ntop(info->se_af, &info->se_dst_addr, dstip_str, + sizeof(dstip_str)); + + sprintf(buf, + "date=%04d-%02d-%02d " + "time=%02d:%02d:%02d " + "timestamp=%lu " + "tz=\"%s\" " + "session_id=%lu " + "src_addr=%s src_port=%d " + "dst_addr=%s dst_port=%d " + "in_bytes=%lu in_pkts=%lu " + "protocol=%d " + "out_bytes=%lu out_pkts=%lu " + "duration=%lu " + "app_name=%s " + "app_proto=%s " + "app_type=%s " + "if_name=\"%s\" ", + ts->tm_year + 1900, ts->tm_mon + 1, ts->tm_mday, ts->tm_hour, + ts->tm_min, ts->tm_sec, utc_now, ts->tm_zone, info->se_id, + srcip_str, ntohs(info->se_src_port), dstip_str, + ntohs(info->se_dst_port), info->se_bytes_in, info->se_pkts_in, + info->se_protocol, info->se_bytes_out, info->se_pkts_out, + info->duration, info->se_app_name ? info->se_app_name : "", + info->se_app_proto ? info->se_app_proto : "", + info->se_app_type ? info->se_app_type : "", + info->se_ifname ? info->se_ifname : ""); +} + +static void export_log_to_file(struct lognode *nodes[], uint32_t size) +{ + FILE *f = fopen(FLOWSTAT_LOG, "a+"); + if (!f) { + if (debug_level >= DEBUG_LEVEL) + RTE_LOG(DEBUG, FLOWSTAT, "Failed to open log file\n"); + return; + } + + char buf[EXPORTER_FILE_BUFSIZ]; + setbuf(f, buf); + + for (uint32_t i = 0; i < size; i++) { + char msg[LOG_MSG_SIZE]; + format_session_log(msg, nodes[i]->info); + fputs(msg, f); + fputs("\n", f); + } + + if (debug_level >= INFO_LEVEL) + RTE_LOG(INFO, FLOWSTAT, "Flushed %d logs\n", size); + + fclose(f); +} + +void export_log(void) +{ + struct lognode *buffer[LOG_BUFFER_LIMIT]; + uint32_t count = 0; + + /* get some limited items from queue */ + while (count < LOG_BUFFER_LIMIT) { + struct cds_wfcq_node *qnode = __cds_wfcq_dequeue_nonblocking( + &logqueue_head, &logqueue_tail); + if (!qnode || qnode == CDS_WFCQ_WOULDBLOCK) + break; /* empty queue or need block */ + + uatomic_dec(&log_buffer_count); + + struct lognode *node = + caa_container_of(qnode, struct lognode, node); + buffer[count++] = node; + } + + if (count > 0) { + /* flush buffer to file */ + export_log_to_file(buffer, count); + + /* clean up */ + for (uint32_t i = 0; i < count; i++) { + struct lognode *node = buffer[i]; + free(node->info); + free(node); + } + } +} + +static void *log_exporter_thread(void *args __unused) +{ + if (debug_level >= INFO_LEVEL) { + unsigned long tid = syscall(SYS_gettid); + RTE_LOG(INFO, FLOWSTAT, "Exporter started threadId=%lu\n", tid); + } + + while (true) { + export_log(); + sleep(EXPORTER_INTERVAL); + } + + if (debug_level >= INFO_LEVEL) + RTE_LOG(INFO, FLOWSTAT, "Exporter stopped\n"); + + return NULL; +} + +static void queue_log(struct dp_session_info *info) +{ + struct lognode *node = malloc(sizeof(struct lognode)); + if (!node) + return; + + cds_wfcq_node_init(&node->node); + node->info = info; + cds_wfcq_enqueue(&logqueue_head, &logqueue_tail, &node->node); + if (debug_level >= DEBUG_LEVEL) { + char msg[LOG_MSG_SIZE]; + format_session_log(msg, info); + RTE_LOG(DEBUG, FLOWSTAT, "Queued log %s\n", msg); + } +} + +static void add_session_log(struct session *s, + struct session_private *s_private, + enum dp_session_state state) +{ + uint32_t next = uatomic_add_return(&log_buffer_count, 1); + if (next > LOG_BUFFER_LIMIT) { + uatomic_dec(&log_buffer_count); + return; + } + + struct dp_session_info *info = malloc(sizeof(struct dp_session_info)); + dp_session_query(s, SESSION_ATTR_ALL, info); + + /* Load cached dpi for expired session */ + if (state == SESSION_STATE_CLOSED) { + info->se_app_name = s_private->app; + info->se_app_proto = s_private->app_proto; + info->se_app_type = s_private->app_type; + } + + time_t now = time(NULL); + info->timestamp = now; + + /* + * duration + */ + int64_t duration = rte_get_timer_cycles() - info->se_create_time; + duration = duration / rte_get_timer_hz(); + if (duration <= 0) + duration = 1; + info->duration = duration; + + if (s_private->is_long_lived) { + uint64_t in_bytes, in_pkts, out_bytes, out_pkts; + + /* calc delta */ + in_bytes = info->se_bytes_in - s_private->last_bytes_in; + in_pkts = info->se_pkts_in - s_private->last_pkts_in; + out_bytes = info->se_bytes_out - s_private->last_bytes_out; + out_pkts = info->se_pkts_out - s_private->last_pkts_out; + + /* no stats was updated, ignore it */ + if (!in_pkts && !out_pkts) + return; + + /* update to latest stat */ + s_private->last_bytes_in = info->se_bytes_in; + s_private->last_pkts_in = info->se_pkts_in; + s_private->last_bytes_out = info->se_bytes_out; + s_private->last_pkts_out = info->se_pkts_out; + + /* save delta for export */ + info->se_bytes_in = in_bytes; + info->se_pkts_in = in_pkts; + info->se_bytes_out = out_bytes; + info->se_pkts_out = out_pkts; + } + + queue_log(info); +} + +static void add_es_session_log(struct session *s, + struct session_private *s_private, + enum dp_session_state state) +{ + /* try acquire lock */ + bool locked = uatomic_xchg(&s_private->locked, true); + if (!locked) { + s_private->is_long_lived = true; + add_session_log(s, s_private, state); + + if (state == SESSION_STATE_ESTABLISHED) + s_private->last_seen = time(NULL); + else if (state == SESSION_STATE_TERMINATING) + s_private->last_seen_tw = time(NULL); + + /* release lock */ + uatomic_set(&s_private->locked, false); + } else if (debug_level >= DEBUG_LEVEL) { + RTE_LOG(DEBUG, FLOWSTAT, "failed to acquire lock es\n"); + } +} + +static void session_watch_cb(struct session *s, enum dp_session_hook hook, + void *data __unused) +{ + /* Check if feature is disabled global */ + if (!is_enabled_global) + return; + + /* Check if feature is enabled for interface */ + struct dp_session_info info; + dp_session_query(s, SESSION_ATTR_IF_NAME | SESSION_ATTR_PROTOCOL, + &info); + + rcu_read_lock(); + bool enabled = intf_lookup(info.se_ifname); + rcu_read_unlock(); + + if (!enabled) + return; + + struct session_private *s_private = NULL; + + /* Set extra data for new session */ + if (hook == SESSION_ACTIVATE) { + s_private = malloc(sizeof(*s_private)); + s_private->locked = false; + s_private->is_long_lived = false; + s_private->last_seen = time(NULL); + s_private->last_seen_tw = 0; + s_private->last_bytes_in = 0; + s_private->last_pkts_in = 0; + s_private->last_bytes_out = 0; + s_private->last_pkts_out = 0; + s_private->app = NULL; + s_private->app_proto = NULL; + s_private->app_type = NULL; + + dp_session_set_private(SESSION_PRIVATE_ID, s, s_private); + } else { + s_private = dp_session_get_private(SESSION_PRIVATE_ID, s); + } + + if (!s_private) + return; + + switch (hook) { + case SESSION_STATS_UPDATE: + if (dp_session_is_established(s)) { + /* For long active connections, export log by interval + */ + double seconds = + difftime(time(NULL), s_private->last_seen); + if (seconds >= LOG_ES_SESSION_INTERVAL) + add_es_session_log(s, s_private, + SESSION_STATE_ESTABLISHED); + } else if (info.se_protocol == IPPROTO_TCP && + info.se_protocol_state >= NPF_TCPS_TIME_WAIT) { + double seconds = + difftime(time(NULL), s_private->last_seen_tw); + if (seconds >= LOG_TW_SESSION_INTERVAL) + add_es_session_log(s, s_private, + SESSION_STATE_TERMINATING); + } + break; + case SESSION_STATE_CHANGE: + if (dp_session_get_state(s) == SESSION_STATE_TERMINATING) { + /* Cache dpi info before session expired. Because + * expired session has no features, mean no dpi info. + */ + if (!s_private->app) { + dp_session_query(s, SESSION_ATTR_DPI, &info); + /* Make it dont try load again if not found any + * dpi + */ + s_private->app = info.se_app_name + ? info.se_app_name + : ""; + s_private->app_proto = info.se_app_proto; + s_private->app_type = info.se_app_type; + } + } + break; + case SESSION_EXPIRE: + /* Closed, add log */ + add_session_log(s, s_private, SESSION_STATE_CLOSED); + + /* Free private data, too */ + free(s_private); + break; + default: + break; + } + + if (debug_level >= DEBUG_LEVEL) { + if (hook == SESSION_STATS_UPDATE) { + char msg[LOG_MSG_SIZE]; + dp_session_query(s, SESSION_ATTR_ALL, &info); + info.timestamp = time(NULL); + info.duration = 0; + format_session_log(msg, &info); + RTE_LOG(DEBUG, FLOWSTAT, + "session uuid=%lu state=%s hook=%d proto=%d " + "proto_state=%d %s\n", + dp_session_unique_id(s), + dp_session_get_state_name(s, false), hook, + info.se_protocol, info.se_protocol_state, msg); + } else { + RTE_LOG(DEBUG, FLOWSTAT, + "session uuid=%lu state=%s hook=%d proto=%d " + "proto_state=%d\n", + dp_session_unique_id(s), + dp_session_get_state_name(s, false), hook, + info.se_protocol, info.se_protocol_state); + } + } +} + +static void fstat_cleanup_cb(const char *instance __unused, + void *context __unused) +{ + cds_wfcq_destroy(&logqueue_head, &logqueue_tail); + cds_lfht_destroy(enabled_intf_ht, NULL); +} + +static unsigned int fstat_process(struct pl_packet *pkt __unused, + void *context __unused) +{ + return FLOW_STAT_ACCEPT; +} + +static int fstat_feat_cmd(struct pb_msg *msg) +{ + int ret = 0; + + FlowStatFeatConfig *fstat_msg = + flow_stat_feat_config__unpack(NULL, msg->msg_len, msg->msg); + if (!fstat_msg) { + dp_pb_cmd_err(msg, "failed to read fstat protobuf command\n"); + return -1; + } + + if (!fstat_msg->has_is_active) { + dp_pb_cmd_err(msg, "error in fstat protobuf command\n"); + return -1; + } + + const char *if_name = fstat_msg->if_name; + + if (strlen(if_name) == 0) { + /* Enable/disable on global */ + if (fstat_msg->is_active == false) { + is_enabled_global = false; + if (debug_level >= INFO_LEVEL) + RTE_LOG(INFO, FLOWSTAT, "disabled global\n"); + } else { + is_enabled_global = true; + if (debug_level >= INFO_LEVEL) + RTE_LOG(INFO, FLOWSTAT, "enabled global\n"); + } + } else if (fstat_msg->is_active == false) { + /* + * Disable interface. + */ + struct intf_node *node = intf_lookup(if_name); + if (node) { + cds_lfht_del(enabled_intf_ht, &node->node); + free(node); + } + + if (debug_level >= INFO_LEVEL) + RTE_LOG(INFO, FLOWSTAT, "disabled on %s\n", if_name); + } else { + /* + * Enable interface. + */ + struct intf_node *node; + struct cds_lfht_node *ret_node; + + node = malloc(sizeof(struct intf_node)); + if (!node) { + dp_pb_cmd_err(msg, "Failed to allocate memory\n"); + return -1; + } + + cds_lfht_node_init(&node->node); + strncpy(node->name, if_name, IFNAMSIZ); + + ret_node = cds_lfht_add_unique( + enabled_intf_ht, intf_hash_name(if_name), intf_match, + if_name, &node->node); + if (ret_node != &node->node) + /* already added, free new node */ + free(node); + + if (debug_level >= INFO_LEVEL) + RTE_LOG(INFO, FLOWSTAT, "enabled on %s\n", if_name); + } + + flow_stat_feat_config__free_unpacked(fstat_msg, NULL); + + return ret; +} + +static int cmd_fstat_feat_show(struct pb_msg *msg) +{ + /* request */ + FlowStatOpReq *fstat_op_req_msg = + flow_stat_op_req__unpack(NULL, msg->msg_len, msg->msg); + if (!fstat_op_req_msg) { + dp_pb_cmd_err(msg, + "failed to read fstat protobuf op command\n"); + return -1; + } + flow_stat_op_req__free_unpacked(fstat_op_req_msg, NULL); + + /* response */ + FlowStatOpResp fstat_op_resp_msg = FLOW_STAT_OP_RESP__INIT; + + fstat_op_resp_msg.count = 0; + fstat_op_resp_msg.has_count = false; + + /* now convert this to binary and add back */ + int len = flow_stat_op_resp__get_packed_size(&fstat_op_resp_msg); + void *buf2 = malloc(len); + flow_stat_op_resp__pack(&fstat_op_resp_msg, buf2); + msg->ret_msg = buf2; + msg->ret_msg_len = len; + + return 0; +} + +const char *fstat_next_nodes[] = { + "vyatta:term-noop", +}; + +static const char *plugin_name = "flow_stat"; + +struct dp_pipeline_feat_registration fstat_feat = { + .plugin_name = "flow_stat", + .name = "fstat:fstat", + .node_name = "fstat:fstat", + .feature_point = "vyatta:ipv4-validate", + .visit_before = NULL, + .visit_after = "vyatta:ipv4-pbr", + .cleanup_cb = fstat_cleanup_cb, +}; + +static struct session_watch fstat_sew = { + .fn = session_watch_cb, + .types = SESSION_TYPE_FW, + .data = NULL, + .name = "fstat_session_watch", +}; + +int dp_feature_plugin_init(const char **name) +{ + int rv; + + rv = dp_pipeline_register_node("fstat:fstat", 1, fstat_next_nodes, + PL_PROC, fstat_process); + if (rv) + goto error; + + rv = dp_pipeline_register_list_feature(&fstat_feat); + if (rv) + goto error; + + rv = dp_feature_register_pb_cfg_handler("fstat:fstat-feat", + fstat_feat_cmd); + if (rv) + goto error; + + rv = dp_feature_register_pb_op_handler("fstat:fstat-feat", + cmd_fstat_feat_show); + if (rv) + goto error; + + rv = dp_session_watch_register(&fstat_sew); + if (rv) + goto error; + + cds_wfcq_init(&logqueue_head, &logqueue_tail); + +#ifndef UNIT_TEST + pthread_t export_thread; + rv = pthread_create(&export_thread, NULL, log_exporter_thread, NULL); + if (rv < 0) { + RTE_LOG(INFO, FLOWSTAT, "pthread create failed\n"); + goto error; + } +#endif + + /* + * Allocate hash table. + */ + enabled_intf_ht = + cds_lfht_new(ENABLED_INTF_HASH_MIN, ENABLED_INTF_HASH_MIN, + ENABLED_INTF_HASH_MAX, + CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, NULL); + if (!enabled_intf_ht) { + RTE_LOG(INFO, FLOWSTAT, "Error allocating hash table\n"); + goto error; + } + + *name = plugin_name; + RTE_LOG(INFO, FLOWSTAT, "flow_stat is loaded\n"); + return 0; +error: + return rv; +} diff --git a/src/pipeline/nodes/flowstat/src/flowstat.h b/src/pipeline/nodes/flowstat/src/flowstat.h new file mode 100644 index 00000000..e88520ed --- /dev/null +++ b/src/pipeline/nodes/flowstat/src/flowstat.h @@ -0,0 +1,24 @@ +/* + * Flow stat pipeline feature node + * + * Copyright (c) 2021, SafePoint. All rights reserved. + * All rights reserved. + * + * SPDX-License-Identifier: LGPL-2.1-only + */ + +#ifndef FLOWSTAT_H +#define FLOWSTAT_H + +#ifdef UNIT_TEST +#define FLOWSTAT_LOG "/tmp/flowstat_test.log" +#define LOG_ES_SESSION_INTERVAL 2 +#else +#define FLOWSTAT_LOG "/var/log/flowstat.log" +#define LOG_ES_SESSION_INTERVAL 60 +#endif + +/* Used by unit-test */ +void export_log(void); + +#endif diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/README.md b/src/pipeline/nodes/flowstat/tests/flowstatd/README.md new file mode 100644 index 00000000..c193c841 --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/README.md @@ -0,0 +1,5 @@ +### RUN TESTS +```sh +export PYTHONPATH=../../scripts +pytest +``` diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/2.log b/src/pipeline/nodes/flowstat/tests/flowstatd/data/2.log new file mode 100644 index 00000000..c7d32157 --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/2.log @@ -0,0 +1,2 @@ +date=2021-03-11 time=03:44:47 timestamp=1615434287 tz="+07" session_id=0 src_addr=192.168.100.79 src_port=55923 dst_addr=192.168.200.9 dst_port=32620 in_bytes=1028 in_pkts=14 protocol=3 out_bytes=303 out_pkts=1 app_name=amqp app_proto=Syslog app_type= if_name="dp0p33p1" +date=2021-03-11 time=03:44:48 timestamp=1615434288 tz="+07" session_id=0 src_addr=192.168.100.214 src_port=50821 dst_addr=192.168.200.157 dst_port=44221 in_bytes=953 in_pkts=9 protocol=13 out_bytes=619 out_pkts=5 app_name=googleplus app_proto=IPP app_type=download if_name="dp0p33p1" diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/gen_log.py b/src/pipeline/nodes/flowstat/tests/flowstatd/data/gen_log.py new file mode 100644 index 00000000..c7ef8624 --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/gen_log.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, SafePoint. All rights reserved. +# +# SPDX-License-Identifier: LGPL-2.1-only +# + +import random +import time +from datetime import datetime, timedelta + + +def gen_log(n, rps=1000, name=None): + APP_NAME_LIST = 'afp aimini ajp amazon amazonvideo amqp apple appleicloud appleitunes applejuice applepush applestore armagetron ayiya battlefield bgp bittorrent bjnp checkmk ciscoskinny ciscovpn citrix cloudflare cnn coap collectd corba crossfire csgo datasaver dce_rpc deezer dhcp dhcpv6 diameter direct_download_link directconnect dnp3 dns dnscrypt dnsoverhttps dofus drda dropbox eaq ebay edonkey egp facebook facebookzero fasttrack fiesta fix florensia ftp_control ftp_data git github gmail gnutella google googledocs googledrive googlehangoutduo googlemaps googleplus googleservices gre gtp guildwars h323 halflife2 hotmail hotspotshield http http_activesync http_connect http_download http_proxy hulu iax icecast icmp icmpv6 iflix igmp imap imaps imo instagram ip_in_ip ipp ipsec irc kakaotalk kakaotalk_voice kerberos kontiki lastfm ldap line linkedin lisp llmnr lotusnotes maplestory mdns megaco memcached messenger mgcp microsoft mining modbus mpeg_ts mqtt ms_onedrive msn mssql_tds mysql nestlogsink netbios netflix netflow nfs nintendo noe ntop ntp ocs office365 ookla opendns openft openvpn oracle oscar ospf pando_media_booster pandora pastebin pcanywhere playstation playstore pop3 pops postgresql pplive ppstream pptp ps_vue qq qqlive quic radius rdp redis remotescan rsync rtcp rtmp rtp rtsp rx sap sctp sflow shoutcast signal sip skype skypecall slack smbv1 smbv23 smpp smtp smtps snapchat snmp socks someip sopcast soulseek soundcloud spotify ssdp ssh starcraft stealthnet steam stun syslog targus_dataspeed teamspeak teamviewer telegram telnet teredo tftp thunder tiktok tinc tls tor truphone tuenti tvants tvuplayer twitch twitter ubntac2 ubuntuone unencrypted_jabber unknown upnp usenet vevo vhua viber vmware vnc vrrp warcraft3 waze webex wechat weibo whatsapp whatsappcall whatsappfiles whois_das wikipedia windowsupdate wireguard worldofkungfu worldofwarcraft xbox xdmcp yahoo youtube youtubeupload zattoo zeromq zoom' + APP_PROTO_LIST = 'FTP_CONTROL POP3 SMTP IMAP DNS IPP HTTP MDNS NTP NetBIOS NFS SSDP BGP SNMP XDMCP SMBv1 Syslog DHCP' + APP_TYPE_LIST = 'chat cloud collaborative database datatransfer download email filesharing game media music network productivity remoteaccess rpc shopping socialnetwork softwareupdate streaming system video voip vpn web' + + APP_NAME_LIST = APP_NAME_LIST.split() + [''] + APP_PROTO_LIST = APP_PROTO_LIST.split() + [''] + APP_TYPE_LIST = APP_TYPE_LIST.split() + [''] + + now = datetime.now() - timedelta(seconds=n) + + if name is None: + name = f'{n}.log' + + with open(name, 'w') as f: + for i in range(n // rps): + logs = [ + f'date={now.strftime("%Y-%m-%d")} ' + f'time={now.strftime("%H:%M:%S")} ' + f'timestamp={int(now.timestamp())} tz="+07" ' + f'session_id={j} ' + f'src_addr=192.168.100.{random.randint(1, 255)} ' + f'src_port={random.randint(30000, 60000)} ' + f'dst_addr=192.168.200.{random.randint(1, 255)} ' + f'dst_port={random.randint(30000, 60000)} ' + f'in_bytes={random.randint(1, 1500)} ' + f'in_pkts={random.randint(1, 15)} ' + f'protocol={random.randint(1, 100)} ' + f'out_bytes={random.randint(1, 1500)} ' + f'out_pkts={random.randint(1, 15)} ' + f'app_name={random.choice(APP_NAME_LIST)} ' + f'app_proto={random.choice(APP_PROTO_LIST)} ' + f'app_type={random.choice(APP_TYPE_LIST)} ' + f'if_name="dp0p33p1"\n' for j in range(rps) + ] + logs = ''.join(logs) + f.write(logs) + now += timedelta(seconds=1) + print('OK') + + +gen_log(2, 1, name='logintf.log') diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/lastfile.log b/src/pipeline/nodes/flowstat/tests/flowstatd/data/lastfile.log new file mode 100644 index 00000000..94f2c5ee --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/lastfile.log @@ -0,0 +1 @@ +date=2021-03-11 time=03:44:47 timestamp=1615434287 tz="+07" session_id=0 src_addr=192.168.100.79 src_port=55923 dst_addr=192.168.200.9 dst_port=32620 in_bytes=1028 in_pkts=14 protocol=3 out_bytes=303 out_pkts=1 app_name=amqp app_proto=Syslog app_type= if_name="dp0p33p1" \ No newline at end of file diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/lastfile2.log b/src/pipeline/nodes/flowstat/tests/flowstatd/data/lastfile2.log new file mode 100644 index 00000000..119867ad --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/lastfile2.log @@ -0,0 +1,2 @@ +date=2021-03-11 time=03:44:47 timestamp=1615434287 tz="+07" session_id=0 src_addr=192.168.100.79 src_port=55923 dst_addr=192.168.200.9 dst_port=32620 in_bytes=1028 in_pkts=14 protocol=3 out_bytes=303 out_pkts=1 app_name=amqp app_proto=Syslog app_type= if_name="dp0p33p1" +date=2021-03-11 time=03:44:48 timestamp=1615434290 tz="+07" session_id=0 src_addr=192.168.100.214 src_port=50821 dst_addr=192.168.200.157 dst_port=44221 in_bytes=953 in_pkts=9 protocol=13 out_bytes=619 out_pkts=5 app_name=googleplus app_proto=IPP app_type=download if_name="dp0p33p1" diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/logintf.log b/src/pipeline/nodes/flowstat/tests/flowstatd/data/logintf.log new file mode 100644 index 00000000..849635fd --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/logintf.log @@ -0,0 +1,2 @@ +date=2021-03-11 time=03:44:47 timestamp=1615434287 tz="+07" session_id=0 src_addr=192.168.100.79 src_port=55923 dst_addr=192.168.200.9 dst_port=32620 in_bytes=1028 in_pkts=14 protocol=3 out_bytes=303 out_pkts=1 app_name=amqp app_proto=Syslog app_type= if_name="dp0p33p1" +date=2021-03-11 time=03:44:48 timestamp=1615434288 tz="+07" session_id=0 src_addr=192.168.100.214 src_port=50821 dst_addr=192.168.200.157 dst_port=44221 in_bytes=953 in_pkts=9 protocol=13 out_bytes=619 out_pkts=5 app_name=googleplus app_proto=IPP app_type=download if_name="dp0p34p1" diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/readlimit.log b/src/pipeline/nodes/flowstat/tests/flowstatd/data/readlimit.log new file mode 100644 index 00000000..119867ad --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/readlimit.log @@ -0,0 +1,2 @@ +date=2021-03-11 time=03:44:47 timestamp=1615434287 tz="+07" session_id=0 src_addr=192.168.100.79 src_port=55923 dst_addr=192.168.200.9 dst_port=32620 in_bytes=1028 in_pkts=14 protocol=3 out_bytes=303 out_pkts=1 app_name=amqp app_proto=Syslog app_type= if_name="dp0p33p1" +date=2021-03-11 time=03:44:48 timestamp=1615434290 tz="+07" session_id=0 src_addr=192.168.100.214 src_port=50821 dst_addr=192.168.200.157 dst_port=44221 in_bytes=953 in_pkts=9 protocol=13 out_bytes=619 out_pkts=5 app_name=googleplus app_proto=IPP app_type=download if_name="dp0p33p1" diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate.log b/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate.log new file mode 100644 index 00000000..f1eaae13 --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate.log @@ -0,0 +1 @@ +date=2021-03-11 time=03:44:48 timestamp=1615434290 tz="+07" session_id=0 src_addr=192.168.100.214 src_port=50821 dst_addr=192.168.200.157 dst_port=44221 in_bytes=953 in_pkts=9 protocol=13 out_bytes=619 out_pkts=5 app_name=googleplus app_proto=IPP app_type=download if_name="dp0p33p1" \ No newline at end of file diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate.log.1 b/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate.log.1 new file mode 100644 index 00000000..94f2c5ee --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate.log.1 @@ -0,0 +1 @@ +date=2021-03-11 time=03:44:47 timestamp=1615434287 tz="+07" session_id=0 src_addr=192.168.100.79 src_port=55923 dst_addr=192.168.200.9 dst_port=32620 in_bytes=1028 in_pkts=14 protocol=3 out_bytes=303 out_pkts=1 app_name=amqp app_proto=Syslog app_type= if_name="dp0p33p1" \ No newline at end of file diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate2.log b/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate2.log new file mode 100644 index 00000000..1c13c670 --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate2.log @@ -0,0 +1 @@ +date=2021-03-11 time=03:44:48 timestamp=1615434288 tz="+07" session_id=0 src_addr=192.168.100.214 src_port=50821 dst_addr=192.168.200.157 dst_port=44221 in_bytes=953 in_pkts=9 protocol=13 out_bytes=619 out_pkts=5 app_name=googleplus app_proto=IPP app_type=download if_name="dp0p33p1" \ No newline at end of file diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate2.log.1 b/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate2.log.1 new file mode 100644 index 00000000..94f2c5ee --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/data/rotate2.log.1 @@ -0,0 +1 @@ +date=2021-03-11 time=03:44:47 timestamp=1615434287 tz="+07" session_id=0 src_addr=192.168.100.79 src_port=55923 dst_addr=192.168.200.9 dst_port=32620 in_bytes=1028 in_pkts=14 protocol=3 out_bytes=303 out_pkts=1 app_name=amqp app_proto=Syslog app_type= if_name="dp0p33p1" \ No newline at end of file diff --git a/src/pipeline/nodes/flowstat/tests/flowstatd/test_flowstatd.py b/src/pipeline/nodes/flowstat/tests/flowstatd/test_flowstatd.py new file mode 100644 index 00000000..4c7c0809 --- /dev/null +++ b/src/pipeline/nodes/flowstat/tests/flowstatd/test_flowstatd.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, SafePoint. All rights reserved. +# +# SPDX-License-Identifier: LGPL-2.1-only +# + +import os +import unittest +from flowstatd import FlowStatDb + +_base_dir = os.path.abspath(__file__ + '/../') + + +def _abs_path(path): + return os.path.abspath(os.path.join(_base_dir, path)) + + +def test_query_top_src(): + db = FlowStatDb(':memory:') + db.connect() + db.collect_from_logfile(_abs_path('data/2.log')) + + res = db.query_top('src', 'bytes', 1615434285, 1615434288) + assert len(res) == 2 + item = res[0] + assert item['key'] == '192.168.100.214' + assert item['in_bytes'] == 953 + assert item['out_bytes'] == 619 + assert item['bytes'] == 1572 + + db.close() + + +def test_query_timeseries(): + db = FlowStatDb(':memory:') + db.connect() + db.collect_from_logfile(_abs_path('data/2.log')) + + res = db.query_time_series('src', 1615434285, 1615434288) + assert len(res) == 1 + item = res[0] + assert item['timestamp'] == 1615434280 + assert item['in_bytes'] == 1981 + assert item['out_bytes'] == 922 + assert item['bytes'] == 2903 + + db.close() + + +def test_query_top_src_intf(): + db = FlowStatDb(':memory:') + db.connect() + db.collect_from_logfile(_abs_path('data/logintf.log')) + + res = db.query_top('src', 'bytes', 1615434285, 1615434288, if_name='dp0p33p1') + assert len(res) == 1 + item = res[0] + assert item['key'] == '192.168.100.79' + assert item['in_bytes'] == 1028 + assert item['out_bytes'] == 303 + assert item['bytes'] == 1331 + + db.close() + + +def test_query_timeseries_intf(): + db = FlowStatDb(':memory:') + db.connect() + db.collect_from_logfile(_abs_path('data/logintf.log')) + + res = db.query_time_series('src', 1615434285, 1615434288, if_name='dp0p33p1') + assert len(res) == 1 + item = res[0] + assert item['timestamp'] == 1615434280 + assert item['in_bytes'] == 1028 + assert item['out_bytes'] == 303 + assert item['bytes'] == 1331 + + db.close() + + +def test_read_limit(): + db = FlowStatDb(':memory:', read_limit=1) + db.connect() + + # read + db.collect_from_logfile(_abs_path('data/2.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 1615434287 + + db.close() + + +def test_read_limit_2(): + db = FlowStatDb(':memory:', read_limit=1) + db.connect() + + # read + db.collect_from_logfile(_abs_path('data/readlimit.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 271 + assert db.last_log['timestamp'] == 1615434290 + + db.close() + + +def test_last_logfile(): + db = FlowStatDb(':memory:') + db.connect() + + # read + db.collect_from_logfile(_abs_path('data/lastfile.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 1615434287 + + db.close() + + +def test_last_log_again(): + db = FlowStatDb(':memory:') + db.connect() + + # read 2 times + db.collect_from_logfile(_abs_path('data/lastfile.log')) + db.collect_from_logfile(_abs_path('data/lastfile.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 1615434287 + + db.close() + + +def test_last_logfile_position(): + db = FlowStatDb(':memory:') + db.connect() + + # read + db.collect_from_logfile(_abs_path('data/lastfile2.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 271 + assert db.last_log['timestamp'] == 1615434290 + + db.close() + + +def test_find_last_logfile(): + db = FlowStatDb(':memory:') + db.connect() + + db._load_metadata() + db.last_log['index'] = 0 + db.last_log['position'] = 0 + db.last_log['timestamp'] = 1615434287 + db._save_metadata() + + db.find_last_logfile(_abs_path('data/lastfile.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 1615434287 + + db.close() + + +def test_find_last_logfile_rotate(): + db = FlowStatDb(':memory:') + db.connect() + + # info of old log file + db.last_log['index'] = 0 + db.last_log['position'] = 0 + db.last_log['timestamp'] = 1615434287 + db._save_metadata() + + db.find_last_logfile(_abs_path('data/rotate.log')) + + assert db.last_log['index'] == 1 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 1615434287 + + db.close() + + +def test_find_last_logfile_wrong_index(): + db = FlowStatDb(':memory:') + db.connect() + + db.last_log['index'] = 1 + db.last_log['position'] = 0 + db.last_log['timestamp'] = 1615434287 + db._save_metadata() + + db.find_last_logfile(_abs_path('data/lastfile.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 1615434287 + + db.close() + + +def test_find_last_logfile_wrong_index_timestamp(): + db = FlowStatDb(':memory:') + db.connect() + + db.last_log['index'] = 1 + db.last_log['position'] = 0 + db.last_log['timestamp'] = 9999 + db._save_metadata() + + db.find_last_logfile(_abs_path('data/lastfile.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 0 + + db.close() + + +def test_find_last_logfile_wrong_position(): + db = FlowStatDb(':memory:') + db.connect() + + db.last_log['index'] = 0 + db.last_log['position'] = 9999 + db.last_log['timestamp'] = 1615434287 + db._save_metadata() + + db.find_last_logfile(_abs_path('data/lastfile.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 0 + + db.close() + + +def test_find_last_logfile_wrong_timestamp(): + db = FlowStatDb(':memory:') + db.connect() + + db.last_log['index'] = 0 + db.last_log['position'] = 0 + db.last_log['timestamp'] = 9999 + db._save_metadata() + + db.find_last_logfile(_abs_path('data/lastfile.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 0 + + db.close() + + +def test_read_two_rotate_logfile_jump_next(): + db = FlowStatDb(':memory:') + db.connect() + + # info of old log file + db.last_log['index'] = 0 + db.last_log['position'] = 0 + db.last_log['timestamp'] = 1615434287 + db._save_metadata() + + # read next + db.collect_from_logfile(_abs_path('data/rotate.log')) + + assert db.last_log['index'] == 0 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 1615434290 + + db.close() + + +def test_read_two_rotate_logfile_not_jump(): + db = FlowStatDb(':memory:') + db.connect() + + # info of old log file + db.last_log['index'] = 0 + db.last_log['position'] = 0 + db.last_log['timestamp'] = 1615434287 + db._save_metadata() + + # read new + db.collect_from_logfile(_abs_path('data/rotate2.log')) + + assert db.last_log['index'] == 1 + assert db.last_log['position'] == 0 + assert db.last_log['timestamp'] == 1615434287 + + db.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/src/pipeline/nodes/flowstat/yang/vyatta-interfaces-flowstat-v1.yang b/src/pipeline/nodes/flowstat/yang/vyatta-interfaces-flowstat-v1.yang new file mode 100644 index 00000000..71292977 --- /dev/null +++ b/src/pipeline/nodes/flowstat/yang/vyatta-interfaces-flowstat-v1.yang @@ -0,0 +1,76 @@ +module vyatta-interfaces-flow-stat-v1 { + namespace "urn:vyatta.com:mgmt:vyatta-interfaces-flow-stat:1"; + prefix vyatta-interfaces-flow-stat-v1; + + import vyatta-interfaces-v1 { + prefix if; + } + import vyatta-interfaces-dataplane-v1 { + prefix interfaces-dataplane; + } + import configd-v1 { + prefix configd; + } + + organization "SafePoint"; + contact + "SafePoint"; + + description + "Copyright (c) 2021 SafePoint. + All rights reserved. + + Copyright (c) 2015-2016 by Brocade Communications Systems, Inc. + All rights reserved. + + Copyright (c) 2019 AT&T Intellectual Property + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + SPDX-License-Identifier: BSD-3-Clause + + The Vyatta configuration/operational YANG for + vyatta-service-sflow-v1."; + + revision 2021-02-25 { + description "Initial revision of version 1."; + } + + augment /if:interfaces/interfaces-dataplane:dataplane { + container flow-stat { + configd:help "Flow statistic"; + leaf enable { + type empty; + configd:help "Enable flow statistic on this interface"; + configd:create "vplane-flowstat.pl --enable=true --intf=$VAR(../../@)"; + configd:delete "vplane-flowstat.pl --enable=false --intf=$VAR(../../@)"; + } + } + } +} diff --git a/src/pipeline/nodes/flowstat/yang/vyatta-op-show-flowstat-v1.yang b/src/pipeline/nodes/flowstat/yang/vyatta-op-show-flowstat-v1.yang new file mode 100644 index 00000000..b6570815 --- /dev/null +++ b/src/pipeline/nodes/flowstat/yang/vyatta-op-show-flowstat-v1.yang @@ -0,0 +1,101 @@ +module vyatta-op-show-flowstat-v1 { + namespace "urn:vyatta.com:mgmt:vyatta-op-show-flowstat:1"; + prefix vyatta-op-show-interfaces-dataplane-v1; + + import vyatta-opd-extensions-v1 { + prefix opd; + } + import vyatta-op-show-v1 { + prefix show; + } + import vyatta-op-show-interfaces-v1 { + prefix if; + } + import vyatta-op-show-dataplane-v1 { + prefix show-dp; + } + + organization "SafePoint."; + contact + "SafePoint"; + + description + "Copyright (c) 2021, SafePoint. + All rights reserved. + + Copyright (c) 2018-2020, AT&T Intellectual Property. + All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, are permitted provided that the following + conditions are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + SPDX-License-Identifier: BSD-3-Clause"; + + revision 2021-03-12 { + description "Initial version"; + } + + opd:augment /show:show/show-dp:dataplane { + opd:command flow-stat { + opd:help "Show dataplane flow statistics"; + opd:inherit "" { + opd:privileged true; + } + + opd:argument if-name { + opd:help "Show flow statistics for a given dataplane interface"; + opd:allowed 'echo ALL; vyatta-interfaces.pl --show dataplane'; + type string; + + opd:argument period { + opd:help "Period"; + opd:allowed "echo now; echo 5m; echo 1h; echo 1d"; + type string; + + opd:argument type { + opd:help "Type"; + opd:allowed "echo top; echo timeseries"; + type string; + + opd:argument key { + opd:help "Key"; + opd:allowed "echo src; echo dst; echo app; echo app_proto; echo app_type"; + type string; + + opd:argument unit { + opd:help "Unit"; + opd:allowed "echo in_bytes; echo out_bytes; echo bytes"; + opd:on-enter 'flowstat_get_state.py --intf=$4 --period=$5 --type=$6 --key=$7 --unit=$8'; + type string; + } + } + } + } + } + } + } +} diff --git a/src/pipeline/nodes/flowstat/yang/vyatta-service-flowstat-v1.yang b/src/pipeline/nodes/flowstat/yang/vyatta-service-flowstat-v1.yang new file mode 100644 index 00000000..5b2caa50 --- /dev/null +++ b/src/pipeline/nodes/flowstat/yang/vyatta-service-flowstat-v1.yang @@ -0,0 +1,102 @@ +module vyatta-service-flow-stat-v1 { + namespace "urn:vyatta.com:mgmt:vyatta-service-flow-stat:1"; + prefix vyatta-service-flow-stat-v1; + + import vyatta-types-v1 { + prefix types; + } + import vyatta-services-v1 { + prefix service; + } + import configd-v1 { + prefix configd; + } + import vyatta-interfaces-v1 { + prefix if; + } + import vyatta-interfaces-dataplane-v1 { + prefix if-dataplane; + } + +organization "SafePoint"; + contact + "SafePoint"; + + description + "Copyright (c) 2021 SafePoint + All rights reserved. + + Copyright (c) 2019, 2021 AT&T Intellectual Property + All rights reserved. + + Copyright (c) 2015-2016 by Brocade Communications Systems, Inc. + All rights reserved. + + The Vyatta configuration YANG for flow-monitoring. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + SPDX-License-Identifier: BSD-3-Clause"; + + revision 2021-03-06 { + description "Initial revision of version 1."; + } + + augment /service:service { + container flow-stat { + configd:help "Flow statistic"; + leaf disable { + type empty; + configd:help "Disable flow statistic global"; + configd:create "vplane-flowstat.pl --cmd=cfg_global --enable=false"; + configd:delete "vplane-flowstat.pl --cmd=cfg_global --enable=true"; + } + + container logrotate { + configd:help "Log file size and rotation characteristics"; + leaf files { + type uint32; + configd:help "Number of log files (default 48)"; + default "48"; + } + leaf size { + type uint32; + configd:help "Size of log files (kbytes) (default 100000)"; + default "100000"; + } + leaf maxage { + type uint32; + configd:help "Remove rotated logs older than (days) (default 2)"; + default "2"; + } + + configd:end "flowstat_update_logrotate"; + } + } + } +} diff --git a/src/session/session.c b/src/session/session.c index 061d56d3..66f9246c 100644 --- a/src/session/session.c +++ b/src/session/session.c @@ -1934,7 +1934,7 @@ int session_npf_pack_restore(struct npf_pack_dp_session *pds, s->se_flags = SESSION_INSERTED; rc = session_npf_pack_stats_restore(s, stats); - if (rc) + if (rc) goto error; *session = s; @@ -2024,3 +2024,79 @@ uint64_t dp_session_unique_id(const struct session *session) return 0; return session->se_id; } + +/* walk function for query features */ +static int se_feature_query(struct session *s, struct session_feature *sf, + void *data) +{ + struct dp_session_info *info = data; + + if (sf->sf_ops && sf->sf_ops->query) + sf->sf_ops->query(info, s, sf); + + return 0; +} + +int dp_session_query(struct session *s, enum dp_session_attr query, + struct dp_session_info *info) +{ + if (!s) + return -1; + + info->se_id = s->se_id; + + if (query & SESSION_ATTR_PROTOCOL) { + info->se_protocol = s->se_protocol; + info->se_protocol_state = s->se_protocol_state; + } + if (query & SESSION_ATTR_BYTES_IN) + info->se_bytes_in = rte_atomic64_read(&s->se_bytes_in); + if (query & SESSION_ATTR_PKTS_IN) + info->se_pkts_in = rte_atomic64_read(&s->se_pkts_in); + if (query & SESSION_ATTR_CREATE_TIME) + info->se_create_time = s->se_create_time; + if (query & SESSION_ATTR_BYTES_OUT) + info->se_bytes_out = rte_atomic64_read(&s->se_bytes_out); + if (query & SESSION_ATTR_PKTS_OUT) + info->se_pkts_out = rte_atomic64_read(&s->se_pkts_out); + + if (query & SESSION_ATTR_SENTRY) { + const void *saddr; + const void *daddr; + uint32_t if_index; + uint16_t sid; + uint16_t did; + + struct sentry *sen = rcu_dereference(s->se_sen); + if (sen) { + session_sentry_extract(sen, &if_index, &info->se_af, + &saddr, &sid, &daddr, &did); + + if (query & SESSION_ATTR_L4_SRC_PORT) + info->se_src_port = sid; + if (query & SESSION_ATTR_IPV4_SRC_ADDR) + info->se_src_addr = *(uint32_t *)saddr; + if (query & SESSION_ATTR_L4_DST_PORT) + info->se_dst_port = did; + if (query & SESSION_ATTR_IPV4_DST_ADDR) + info->se_dst_addr = *(uint32_t *)daddr; + if (query & SESSION_ATTR_IF_NAME) + info->se_ifname = + ifnet_indextoname_safe(if_index); + } + } + + if (query & SESSION_ATTR_DPI) { + info->query = query; + + /* set default value in case no info found */ + info->se_app_name = NULL; + info->se_app_proto = NULL; + info->se_app_type = NULL; + + session_feature_walk_session(s, SESSION_FEATURE_ALL, + se_feature_query, info); + } + + return 0; +} diff --git a/src/session/session.h b/src/session/session.h index fc352c8a..2ab5632b 100644 --- a/src/session/session.h +++ b/src/session/session.h @@ -106,6 +106,9 @@ struct session_feature_ops { void (*log)(enum session_log_event event, struct session *s, struct session_feature *sf); int (*nat_info)(void *data, uint32_t *taddr, uint16_t *tport); + + void (*query)(struct dp_session_info *info, + struct session *s, struct session_feature *sf); }; #define SESS_FEAT_REQ_EXPIRY 0x01 /* feature marked for expiry */ diff --git a/tests/whole_dp/meson.build b/tests/whole_dp/meson.build index ee111c8b..ad413c84 100644 --- a/tests/whole_dp/meson.build +++ b/tests/whole_dp/meson.build @@ -212,6 +212,27 @@ sample_test_plugin = shared_module('sample_test', dependencies: [dpdk_dep, protobuf_generated_c_dependency] ) +flowstat_test_plugin_sources = files( + 'src/dp_test_pipeline_flowstat.c', +) + +flowstat_test_plugin = shared_module('flowstat_test', + sources: [ + flowstat_test_plugin_sources, + flowstat_generated_protobuf_c, + flowstat_plugin_sources, + ], + include_directories: [ + public_include, + public_test_include, + internal_inc, + flowstat_include, + internal_test_inc, + ], + dependencies: [dpdk_dep, protobuf_generated_c_dependency], + c_args: ['-DUNIT_TEST', '-Wno-unused-function'] +) + fal_test_plugin_sources = files( 'src/fal_plugin_test.c', 'src/fal_plugin_sw_port.c', @@ -268,14 +289,20 @@ endif lcore_number = 0 cores_available = run_command('nproc').stdout().to_int() -foreach suite : check_tests + ['dp_test_pipeline.c'] +foreach suite : check_tests + ['dp_test_pipeline.c', 'dp_test_pipeline_flowstat.c'] suite_env = ['CK_RUN_SUITE=@0@'.format(suite), 'CK_XML_LOG_FILE_NAME=test_@0@.xml'.format(suite)] + dataplane_test_env + if suite == 'dp_test_pipeline_flowstat.c' + test_fea_plugin_path = meson.current_build_dir() + else + test_fea_plugin_path = meson.build_root() / 'src/pipeline/nodes/sample' + endif + test(suite, dataplane_test, - depends: [sample_plugin, sample_test_plugin, fal_test_plugin, dummyfs], + depends: [sample_plugin, sample_test_plugin, flowstat_plugin, flowstat_test_plugin, fal_test_plugin, dummyfs], workdir: meson.current_build_dir(), - args: ['-l @0@'.format(lcore_number), '-d1', '-F', meson.build_root() / 'src/pipeline/nodes/sample', '-P', meson.current_build_dir()], + args: ['-l @0@'.format(lcore_number), '-d1', '-F', test_fea_plugin_path, '-P', meson.current_build_dir()], env: suite_env, timeout: test_timeout ) diff --git a/tests/whole_dp/src/dp_test_pipeline_flowstat.c b/tests/whole_dp/src/dp_test_pipeline_flowstat.c new file mode 100644 index 00000000..fdfc038a --- /dev/null +++ b/tests/whole_dp/src/dp_test_pipeline_flowstat.c @@ -0,0 +1,1224 @@ +/* + * Copyright (c) 2021, SafePoint. All rights reserved. + * Copyright (c) 2017-2020, AT&T Intellectual Property. All rights reserved. + * Copyright (c) 2015-2017 by Brocade Communications Systems, Inc. + * All rights reserved. + * + * SPDX-License-Identifier: LGPL-2.1-only + * + * Whole dataplane test pipeline flowstat tests + */ + +#include "compiler.h" +#include "dp_test/dp_test_lib.h" +#include "dp_test/dp_test_lib_intf.h" +#include "dp_test/dp_test_macros.h" +#include "dp_test/dp_test_pktmbuf_lib.h" +#include "dp_test/dp_test_netlink_state.h" +#include "dp_test/dp_test_firewall_lib.h" +#include "dp_test/dp_test_session_lib.h" +#include "dp_test_lib_tcp.h" +#include "dp_test_npf_lib.h" + +#include "dp_test_npf_sess_lib.h" +#include "dp_test_session_internal_lib.h" + +#include + +#include "flowstat.h" +#include "FlowStatFeatConfig.pb-c.h" +#include "FlowStatFeatOp.pb-c.h" +#include "protobuf/DataplaneEnvelope.pb-c.h" + +#include "pcap/http-example.h" +#include "pcap/http-google.h" + +struct dp_test_flow_log_result { + const char *src; + int src_port; + const char *dst; + int dst_port; + int protocol; + uint64_t in_pkts; + uint64_t out_pkts; + const char *app; + const char *app_proto; + const char *app_type; + const char *if_name; +}; + +static struct dpt_tcp_flow_pkt tcp_simple_pkt[] = { + {DPT_FORW, TH_SYN, 0, NULL, 0, NULL}, + {DPT_BACK, TH_SYN | TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 20, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 50, NULL, 0, NULL}, + {DPT_BACK, TH_FIN, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_FIN, 0, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, +}; + +void _dp_test_remove_log_file(void) +{ + remove(FLOWSTAT_LOG); +} + +void _dp_test_verify_log(const char *log, + const struct dp_test_flow_log_result *p, + const char *file, int line) +{ + char buf[100]; + + sprintf(buf, "src_addr=%s ", p->src); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "src_port=%d ", p->src_port); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "dst_addr=%s ", p->dst); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "dst_port=%d ", p->dst_port); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "in_pkts=%lu ", p->in_pkts); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "out_pkts=%lu ", p->out_pkts); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "protocol=%d ", p->protocol); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "app_name=%s ", p->app); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "app_proto=%s ", p->app_proto); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "app_type=%s ", p->app_type); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); + + sprintf(buf, "if_name=\"%s\"", p->if_name); + _dp_test_fail_unless(strstr(log, buf) != NULL, file, line, + "Expected %s", buf); +} + +#define dp_test_verify_log(log, log_result) \ + _dp_test_verify_log(log, log_result, __FILE__, __LINE__) + +void _dp_test_verify_log_file(const struct dp_test_flow_log_result *log_result, + size_t size, + const char *file, int line) +{ + char log_line[1000]; + + FILE *f = fopen(FLOWSTAT_LOG, "r"); + _dp_test_fail_unless(f != NULL, file, line, + "Expected have log file %s", FLOWSTAT_LOG); + + for (size_t i = 0; i < size; i++) { + char *rv = fgets(log_line, 1000, f); + _dp_test_fail_unless(rv != NULL, file, line, + "Expected a log line"); + _dp_test_verify_log(log_line, &log_result[i], file, line); + } + + fclose(f); +} + +#define dp_test_verify_log_file(log_result, size) \ + _dp_test_verify_log_file(log_result, size, __FILE__, __LINE__) + +void _dp_test_verify_no_log_file(const char *file, int line) +{ + FILE *f = fopen(FLOWSTAT_LOG, "rb"); + _dp_test_fail_unless(f == NULL, file, line, + "Expected no log file %s", FLOWSTAT_LOG); +} + +#define dp_test_verify_no_log_file() \ + _dp_test_verify_no_log_file(__FILE__, __LINE__) + +static void +dp_test_create_and_send_cfg_feat_msg(bool enable, const char *ifname) +{ + int len; + FlowStatFeatConfig cfg = FLOW_STAT_FEAT_CONFIG__INIT; + + /* set values here */ + cfg.is_active = enable; + cfg.has_is_active = true; + /* strings don't have 'has_' */ + cfg.if_name = (char *)ifname; + len = flow_stat_feat_config__get_packed_size(&cfg); + void *buf2 = malloc(len); + assert(buf2); + + flow_stat_feat_config__pack(&cfg, buf2); + + dp_test_lib_pb_wrap_and_send_pb("fstat:fstat-feat", buf2, len); +} + +DP_DECL_TEST_SUITE(flowstat); + +DP_DECL_TEST_CASE(flowstat, general, NULL, NULL); + +/* + * Test not enabled + */ +DP_START_TEST(general, not_enabled) +{ + char *dp1T0_mac = dp_test_intf_name2mac_str("dp1T0"); + char *dp2T1_mac = dp_test_intf_name2mac_str("dp2T1"); + + /* Setup interfaces and neighbours */ + dp_test_nl_add_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_add_ip_addr_and_connected("dp2T1", "200.201.202.1/24"); + + dp_test_netlink_add_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_add_neigh("dp2T1", "200.201.202.203", + "aa:bb:cc:18:0:1"); + + _dp_test_remove_log_file(); + + struct dp_test_pkt_desc_t *ins_pre, *ins_post; + struct dp_test_pkt_desc_t *outs_pre, *outs_post; + + ins_pre = dpt_pdesc_v4_create( + "Inside pre", IPPROTO_TCP, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + dp1T0_mac, "200.201.202.203", 80, + "dp1T0", "dp2T1"); + + ins_post = dpt_pdesc_v4_create( + "Inside post", IPPROTO_TCP, + dp2T1_mac, "100.101.102.103", 49152, + "aa:bb:cc:18:0:1", "200.201.202.203", 80, + "dp1T0", "dp2T1"); + + outs_pre = dpt_pdesc_v4_create( + "Outside pre", IPPROTO_TCP, + "aa:bb:cc:18:0:1", "200.201.202.203", 80, + dp2T1_mac, "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + outs_post = dpt_pdesc_v4_create( + "Outside post", IPPROTO_TCP, + dp1T0_mac, "200.201.202.203", 80, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + struct dp_test_npf_rule_t rules[] = { + { + .rule = "10", + .pass = PASS, + .stateful = STATEFUL, + .npf = "to=any" + }, + RULE_DEF_BLOCK, + NULL_RULE + }; + + struct dp_test_npf_ruleset_t fw = { + .rstype = "fw-out", + .name = "FW1_OUT", + .enable = 1, + .attach_point = "dp2T1", + .fwd = FWD, + .dir = "out", + .rules = rules + }; + + dp_test_npf_fw_add(&fw, false); + + struct dpt_tcp_flow tcp_call = { + .text[0] = '\0', + .isn = {0, 0}, + .desc[DPT_FORW] = { + .pre = ins_pre, + .pst = ins_post, + }, + .desc[DPT_BACK] = { + .pre = outs_pre, + .pst = outs_post, + }, + .test_cb = NULL, + .post_cb = NULL, + }; + + dpt_tcp_call(&tcp_call, tcp_simple_pkt, ARRAY_SIZE(tcp_simple_pkt), + 0, 0, NULL, 0); + + /* Simulate export log */ + export_log(); + + dp_test_verify_no_log_file(); + + /* + * End + */ + + free(ins_pre); + free(ins_post); + free(outs_pre); + free(outs_post); + + /************************************************************* + * Cleanup + *************************************************************/ + + dp_test_npf_fw_del(&fw, false); + dp_test_npf_clear_sessions(); + + dp_test_netlink_del_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_del_neigh("dp2T1", "200.201.202.203", + "aa:bb:cc:18:0:1"); + + dp_test_nl_del_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_del_ip_addr_and_connected("dp2T1", "200.201.202.1/24"); + +} DP_END_TEST; + +DP_DECL_TEST_CASE(flowstat, logflow, NULL, NULL); + +/* + * Test Log simple tcp session + */ +DP_START_TEST(logflow, simple) +{ + char *dp1T0_mac = dp_test_intf_name2mac_str("dp1T0"); + char *dp2T1_mac = dp_test_intf_name2mac_str("dp2T1"); + + /* Setup interfaces and neighbours */ + dp_test_nl_add_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_add_ip_addr_and_connected("dp2T1", "200.201.202.1/24"); + + dp_test_netlink_add_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_add_neigh("dp2T1", "200.201.202.203", + "aa:bb:cc:18:0:1"); + + _dp_test_remove_log_file(); + + /* Enable the feature */ + char real_ifname[IFNAMSIZ]; + dp_test_create_and_send_cfg_feat_msg( + true, dp_test_intf_real("dp2T1", real_ifname)); + + struct dp_test_pkt_desc_t *ins_pre, *ins_post; + struct dp_test_pkt_desc_t *outs_pre, *outs_post; + + ins_pre = dpt_pdesc_v4_create( + "Inside pre", IPPROTO_TCP, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + dp1T0_mac, "200.201.202.203", 80, + "dp1T0", "dp2T1"); + + ins_post = dpt_pdesc_v4_create( + "Inside post", IPPROTO_TCP, + dp2T1_mac, "100.101.102.103", 49152, + "aa:bb:cc:18:0:1", "200.201.202.203", 80, + "dp1T0", "dp2T1"); + + outs_pre = dpt_pdesc_v4_create( + "Outside pre", IPPROTO_TCP, + "aa:bb:cc:18:0:1", "200.201.202.203", 80, + dp2T1_mac, "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + outs_post = dpt_pdesc_v4_create( + "Outside post", IPPROTO_TCP, + dp1T0_mac, "200.201.202.203", 80, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + struct dp_test_npf_rule_t rules[] = { + { + .rule = "10", + .pass = PASS, + .stateful = STATEFUL, + .npf = "to=any" + }, + RULE_DEF_BLOCK, + NULL_RULE + }; + + struct dp_test_npf_ruleset_t fw = { + .rstype = "fw-out", + .name = "FW1_OUT", + .enable = 1, + .attach_point = "dp2T1", + .fwd = FWD, + .dir = "out", + .rules = rules + }; + + dp_test_npf_fw_add(&fw, false); + + struct dpt_tcp_flow tcp_call = { + .text[0] = '\0', + .isn = {0, 0}, + .desc[DPT_FORW] = { + .pre = ins_pre, + .pst = ins_post, + }, + .desc[DPT_BACK] = { + .pre = outs_pre, + .pst = outs_post, + }, + .test_cb = NULL, + .post_cb = NULL, + }; + + dpt_tcp_call(&tcp_call, tcp_simple_pkt, ARRAY_SIZE(tcp_simple_pkt), + 0, 0, NULL, 0); + + /* Simulate export log */ + export_log(); + + struct dp_test_flow_log_result log_result = { + .src = "100.101.102.103", + .src_port = 49152, + .dst = "200.201.202.203", + .dst_port = 80, + .protocol = IPPROTO_TCP, + .in_pkts = 5, + .out_pkts = 5, + .app = "", + .app_proto = "", + .app_type = "", + .if_name = dp_test_intf_real("dp2T1", real_ifname), + }; + dp_test_verify_log_file(&log_result, 1); + + /* + * End + */ + + free(ins_pre); + free(ins_post); + free(outs_pre); + free(outs_post); + + /************************************************************* + * Cleanup + *************************************************************/ + + dp_test_create_and_send_cfg_feat_msg( + false, dp_test_intf_real("dp2T1", real_ifname)); + dp_test_npf_fw_del(&fw, false); + dp_test_npf_clear_sessions(); + + dp_test_netlink_del_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_del_neigh("dp2T1", "200.201.202.203", + "aa:bb:cc:18:0:1"); + + dp_test_nl_del_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_del_ip_addr_and_connected("dp2T1", "200.201.202.1/24"); + +} DP_END_TEST; + +/* + * Test log long lived tcp session + */ +static void +dp_test_logflow_ll_tcp_post_cb(uint pktno, bool forw __unused, + uint8_t flags __unused, + struct dp_test_pkt_desc_t *pre __unused, + struct dp_test_pkt_desc_t *post __unused, + const char *desc __unused) +{ + if (pktno == 3) + sleep(2); +} + +DP_START_TEST(logflow, long_lived_tcp) +{ + char *dp1T0_mac = dp_test_intf_name2mac_str("dp1T0"); + char *dp2T1_mac = dp_test_intf_name2mac_str("dp2T1"); + + /* Setup interfaces and neighbours */ + dp_test_nl_add_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_add_ip_addr_and_connected("dp2T1", "200.201.202.1/24"); + + dp_test_netlink_add_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_add_neigh("dp2T1", "200.201.202.203", + "aa:bb:cc:18:0:1"); + + _dp_test_remove_log_file(); + + /* Enable the feature */ + char real_ifname[IFNAMSIZ]; + dp_test_create_and_send_cfg_feat_msg( + true, dp_test_intf_real("dp2T1", real_ifname)); + + struct dp_test_pkt_desc_t *ins_pre, *ins_post; + struct dp_test_pkt_desc_t *outs_pre, *outs_post; + + ins_pre = dpt_pdesc_v4_create( + "Inside pre", IPPROTO_TCP, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + dp1T0_mac, "200.201.202.203", 80, + "dp1T0", "dp2T1"); + + ins_post = dpt_pdesc_v4_create( + "Inside post", IPPROTO_TCP, + dp2T1_mac, "100.101.102.103", 49152, + "aa:bb:cc:18:0:1", "200.201.202.203", 80, + "dp1T0", "dp2T1"); + + outs_pre = dpt_pdesc_v4_create( + "Outside pre", IPPROTO_TCP, + "aa:bb:cc:18:0:1", "200.201.202.203", 80, + dp2T1_mac, "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + outs_post = dpt_pdesc_v4_create( + "Outside post", IPPROTO_TCP, + dp1T0_mac, "200.201.202.203", 80, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + struct dp_test_npf_rule_t rules[] = { + { + .rule = "10", + .pass = PASS, + .stateful = STATEFUL, + .npf = "to=any" + }, + RULE_DEF_BLOCK, + NULL_RULE + }; + + struct dp_test_npf_ruleset_t fw = { + .rstype = "fw-out", + .name = "FW1_OUT", + .enable = 1, + .attach_point = "dp2T1", + .fwd = FWD, + .dir = "out", + .rules = rules + }; + + dp_test_npf_fw_add(&fw, false); + + struct dpt_tcp_flow tcp_call = { + .text[0] = '\0', + .isn = {0, 0}, + .desc[DPT_FORW] = { + .pre = ins_pre, + .pst = ins_post, + }, + .desc[DPT_BACK] = { + .pre = outs_pre, + .pst = outs_post, + }, + .test_cb = NULL, + .post_cb = dp_test_logflow_ll_tcp_post_cb, + }; + + struct dpt_tcp_flow_pkt tcp_pkt1[] = { + {DPT_FORW, TH_SYN, 0, NULL, 0, NULL}, + {DPT_BACK, TH_SYN | TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 20, NULL, 0, NULL}, + /* long lived session */ + {DPT_FORW, TH_ACK, 50, NULL, 0, NULL}, + {DPT_BACK, TH_FIN, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_FIN, 0, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, + }; + + dpt_tcp_call(&tcp_call, tcp_pkt1, ARRAY_SIZE(tcp_pkt1), 0, 0, NULL, 0); + + /* Simulate export log */ + export_log(); + + struct dp_test_flow_log_result log_result[] = {{ + .src = "100.101.102.103", + .src_port = 49152, + .dst = "200.201.202.203", + .dst_port = 80, + .protocol = IPPROTO_TCP, + .in_pkts = 2, + .out_pkts = 3, + .app = "", + .app_proto = "", + .app_type = "", + .if_name = dp_test_intf_real("dp2T1", real_ifname), + }, { + .src = "100.101.102.103", + .src_port = 49152, + .dst = "200.201.202.203", + .dst_port = 80, + .protocol = IPPROTO_TCP, + .in_pkts = 3, + .out_pkts = 2, + .app = "", + .app_proto = "", + .app_type = "", + .if_name = dp_test_intf_real("dp2T1", real_ifname), + }}; + dp_test_verify_log_file(log_result, ARRAY_SIZE(log_result)); + + /* + * End + */ + + free(ins_pre); + free(ins_post); + free(outs_pre); + free(outs_post); + + /************************************************************* + * Cleanup + *************************************************************/ + + dp_test_create_and_send_cfg_feat_msg( + false, dp_test_intf_real("dp2T1", real_ifname)); + dp_test_npf_fw_del(&fw, false); + dp_test_npf_clear_sessions(); + + dp_test_netlink_del_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_del_neigh("dp2T1", "200.201.202.203", + "aa:bb:cc:18:0:1"); + + dp_test_nl_del_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_del_ip_addr_and_connected("dp2T1", "200.201.202.1/24"); + +} DP_END_TEST; + +/* + * 100.101.102.x 216.58.200.x + * +----------+ + * .1| |.1 + * .103 -------------------| UUT |------------------ .78 (google.com) + * dp1T0| |dp2T1 + * +----------+ + * + * + * Test http get google.com + */ +DP_START_TEST(logflow, google) +{ + char *dp1T0_mac = dp_test_intf_name2mac_str("dp1T0"); + char *dp2T1_mac = dp_test_intf_name2mac_str("dp2T1"); + + /* Setup interfaces and neighbours */ + dp_test_nl_add_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_add_ip_addr_and_connected("dp2T1", "216.58.200.1/24"); + + dp_test_netlink_add_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_add_neigh("dp2T1", "216.58.200.78", + "aa:bb:cc:18:0:1"); + + _dp_test_remove_log_file(); + + /* Enable the feature */ + char real_ifname[IFNAMSIZ]; + dp_test_create_and_send_cfg_feat_msg( + true, dp_test_intf_real("dp2T1", real_ifname)); + + struct dp_test_pkt_desc_t *ins_pre, *ins_post; + struct dp_test_pkt_desc_t *outs_pre, *outs_post; + + ins_pre = dpt_pdesc_v4_create( + "Inside pre", IPPROTO_TCP, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + dp1T0_mac, "216.58.200.78", 80, + "dp1T0", "dp2T1"); + + ins_post = dpt_pdesc_v4_create( + "Inside post", IPPROTO_TCP, + dp2T1_mac, "100.101.102.103", 49152, + "aa:bb:cc:18:0:1", "216.58.200.78", 80, + "dp1T0", "dp2T1"); + + outs_pre = dpt_pdesc_v4_create( + "Outside pre", IPPROTO_TCP, + "aa:bb:cc:18:0:1", "216.58.200.78", 80, + dp2T1_mac, "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + outs_post = dpt_pdesc_v4_create( + "Outside post", IPPROTO_TCP, + dp1T0_mac, "216.58.200.78", 80, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + struct dp_test_npf_rule_t rules[] = { + { + .rule = "10", + .pass = PASS, + .stateful = STATEFUL, + .npf = "to=any" + }, + RULE_DEF_BLOCK, + NULL_RULE + }; + + struct dp_test_npf_ruleset_t fw = { + .rstype = "fw-out", + .name = "FW1_OUT", + .enable = 1, + .attach_point = "dp2T1", + .fwd = FWD, + .dir = "out", + .rules = rules + }; + + dp_test_npf_fw_add(&fw, false); + + struct dpt_tcp_flow tcp_call = { + .text[0] = '\0', + .isn = {0, 0}, + .desc[DPT_FORW] = { + .pre = ins_pre, + .pst = ins_post, + }, + .desc[DPT_BACK] = { + .pre = outs_pre, + .pst = outs_post, + }, + .test_cb = NULL, + .post_cb = NULL, + }; + + dpt_tcp_call(&tcp_call, http_google_pkt, ARRAY_SIZE(http_google_pkt), + 0, 0, NULL, 0); + + /* Simulate export log */ + export_log(); + + struct dp_test_flow_log_result log_result = { + .src = "100.101.102.103", + .src_port = 49152, + .dst = "216.58.200.78", + .dst_port = 80, + .protocol = IPPROTO_TCP, + .in_pkts = 8, + .out_pkts = 8, + .app = "", + .app_proto = "", + .app_type = "", + .if_name = dp_test_intf_real("dp2T1", real_ifname), + }; + dp_test_verify_log_file(&log_result, 1); + + /* + * End + */ + + free(ins_pre); + free(ins_post); + free(outs_pre); + free(outs_post); + + /************************************************************* + * Cleanup + *************************************************************/ + + dp_test_create_and_send_cfg_feat_msg( + false, dp_test_intf_real("dp2T1", real_ifname)); + dp_test_npf_fw_del(&fw, false); + dp_test_npf_clear_sessions(); + + dp_test_netlink_del_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_del_neigh("dp2T1", "216.58.200.78", + "aa:bb:cc:18:0:1"); + + dp_test_nl_del_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_del_ip_addr_and_connected("dp2T1", "216.58.200.1/24"); + +} DP_END_TEST; + +DP_DECL_TEST_CASE(flowstat, logflowdpi, NULL, NULL); + +/* + * 100.101.102.x 93.184.216.x + * +----------+ + * .1| |.1 + * .103 -------------------| UUT |------------------ .34 (example.com) + * dp1T0| |dp2T1 + * +----------+ + * + * + * http get example.com (with DPI) + */ +DP_START_TEST(logflowdpi, example) +{ + char *dp1T0_mac = dp_test_intf_name2mac_str("dp1T0"); + char *dp2T1_mac = dp_test_intf_name2mac_str("dp2T1"); + + /* Setup interfaces and neighbours */ + dp_test_nl_add_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_add_ip_addr_and_connected("dp2T1", "93.184.216.1/24"); + + dp_test_netlink_add_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_add_neigh("dp2T1", "93.184.216.34", + "aa:bb:cc:18:0:1"); + + _dp_test_remove_log_file(); + + /* Enable the feature */ + char real_ifname[IFNAMSIZ]; + dp_test_create_and_send_cfg_feat_msg( + true, dp_test_intf_real("dp2T1", real_ifname)); + + struct dp_test_pkt_desc_t *ins_pre, *ins_post; + struct dp_test_pkt_desc_t *outs_pre, *outs_post; + + ins_pre = dpt_pdesc_v4_create( + "Inside pre", IPPROTO_TCP, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + dp1T0_mac, "93.184.216.34", 80, + "dp1T0", "dp2T1"); + + ins_post = dpt_pdesc_v4_create( + "Inside post", IPPROTO_TCP, + dp2T1_mac, "100.101.102.103", 49152, + "aa:bb:cc:18:0:1", "93.184.216.34", 80, + "dp1T0", "dp2T1"); + + outs_pre = dpt_pdesc_v4_create( + "Outside pre", IPPROTO_TCP, + "aa:bb:cc:18:0:1", "93.184.216.34", 80, + dp2T1_mac, "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + outs_post = dpt_pdesc_v4_create( + "Outside post", IPPROTO_TCP, + dp1T0_mac, "93.184.216.34", 80, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + /* + * dpi app firewall + */ + dp_test_npf_cmd_fmt(false, + "npf-ut delete app-firewall:ALLOWED-SITES 10000"); + dp_test_npf_cmd_fmt(false, "npf-ut add app-firewall:ALLOWED-SITES 100 " + "action=accept engine=ndpi protocol=http"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, "npf-ut add fw:DPI 200 action=accept " + "proto-final=6 stateful=y dst-port=80 " + "rproc=app-firewall(ALLOWED-SITES)"); + dp_test_npf_cmd_fmt(false, + "npf-ut add fw:DPI 1000 action=accept stateful=y"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, + "npf-ut attach interface:dpT21 fw-out fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + + struct dpt_tcp_flow tcp_call = { + .text[0] = '\0', + .isn = {0, 0}, + .desc[DPT_FORW] = { + .pre = ins_pre, + .pst = ins_post, + }, + .desc[DPT_BACK] = { + .pre = outs_pre, + .pst = outs_post, + }, + .test_cb = NULL, + .post_cb = NULL, + }; + + dpt_tcp_call(&tcp_call, http_example_pkt, ARRAY_SIZE(http_example_pkt), + 0, 0, NULL, 0); + + /* Simulate export log */ + export_log(); + + struct dp_test_flow_log_result log_result = { + .src = "100.101.102.103", + .src_port = 49152, + .dst = "93.184.216.34", + .dst_port = 80, + .protocol = IPPROTO_TCP, + .in_pkts = 8, + .out_pkts = 8, + .app = "HTTP", + .app_proto = "HTTP", + .app_type = "Web", + .if_name = dp_test_intf_real("dp2T1", real_ifname), + }; + dp_test_verify_log_file(&log_result, 1); + + /* + * End + */ + + free(ins_pre); + free(ins_post); + free(outs_pre); + free(outs_post); + + /************************************************************* + * Cleanup + *************************************************************/ + + dp_test_create_and_send_cfg_feat_msg( + false, dp_test_intf_real("dp2T1", real_ifname)); + + dp_test_npf_cmd_fmt(false, + "npf-ut delete app-firewall:ALLOWED-SITES 10000"); + dp_test_npf_cmd_fmt(false, "npf-ut delete app-firewall:ALLOWED-SITES"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, + "npf-ut detach interface:dpT21 fw-out fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, "npf-ut delete fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + + dp_test_npf_clear_sessions(); + + dp_test_netlink_del_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_del_neigh("dp2T1", "93.184.216.34", + "aa:bb:cc:18:0:1"); + + dp_test_nl_del_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_del_ip_addr_and_connected("dp2T1", "93.184.216.1/24"); + +} DP_END_TEST; + +/* + * 100.101.102.x 216.58.200.x + * +----------+ + * .1| |.1 + * .103 -------------------| UUT |------------------ .78 (google.com) + * dp1T0| |dp2T1 + * +----------+ + * + * + * http get google.com (with DPI) + */ +DP_START_TEST(logflowdpi, google) +{ + char *dp1T0_mac = dp_test_intf_name2mac_str("dp1T0"); + char *dp2T1_mac = dp_test_intf_name2mac_str("dp2T1"); + + /* Setup interfaces and neighbours */ + dp_test_nl_add_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_add_ip_addr_and_connected("dp2T1", "216.58.200.1/24"); + + dp_test_netlink_add_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_add_neigh("dp2T1", "216.58.200.78", + "aa:bb:cc:18:0:1"); + + _dp_test_remove_log_file(); + + /* Enable the feature */ + char real_ifname[IFNAMSIZ]; + dp_test_create_and_send_cfg_feat_msg( + true, dp_test_intf_real("dp2T1", real_ifname)); + + struct dp_test_pkt_desc_t *ins_pre, *ins_post; + struct dp_test_pkt_desc_t *outs_pre, *outs_post; + + ins_pre = dpt_pdesc_v4_create( + "Inside pre", IPPROTO_TCP, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + dp1T0_mac, "216.58.200.78", 80, + "dp1T0", "dp2T1"); + + ins_post = dpt_pdesc_v4_create( + "Inside post", IPPROTO_TCP, + dp2T1_mac, "100.101.102.103", 49152, + "aa:bb:cc:18:0:1", "216.58.200.78", 80, + "dp1T0", "dp2T1"); + + outs_pre = dpt_pdesc_v4_create( + "Outside pre", IPPROTO_TCP, + "aa:bb:cc:18:0:1", "216.58.200.78", 80, + dp2T1_mac, "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + outs_post = dpt_pdesc_v4_create( + "Outside post", IPPROTO_TCP, + dp1T0_mac, "216.58.200.78", 80, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + /* + * dpi app firewall + */ + dp_test_npf_cmd_fmt(false, + "npf-ut delete app-firewall:ALLOWED-SITES 10000"); + dp_test_npf_cmd_fmt(false, "npf-ut add app-firewall:ALLOWED-SITES 100 " + "action=accept engine=ndpi protocol=http"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, "npf-ut add fw:DPI 200 action=accept " + "proto-final=6 stateful=y dst-port=80 " + "rproc=app-firewall(ALLOWED-SITES)"); + dp_test_npf_cmd_fmt(false, + "npf-ut add fw:DPI 1000 action=accept stateful=y"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, + "npf-ut attach interface:dpT21 fw-out fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + + struct dpt_tcp_flow tcp_call = { + .text[0] = '\0', + .isn = {0, 0}, + .desc[DPT_FORW] = { + .pre = ins_pre, + .pst = ins_post, + }, + .desc[DPT_BACK] = { + .pre = outs_pre, + .pst = outs_post, + }, + .test_cb = NULL, + .post_cb = NULL, + }; + + dpt_tcp_call(&tcp_call, http_google_pkt, ARRAY_SIZE(http_google_pkt), + 0, 0, NULL, 0); + + /* Simulate export log */ + export_log(); + + struct dp_test_flow_log_result log_result = { + .src = "100.101.102.103", + .src_port = 49152, + .dst = "216.58.200.78", + .dst_port = 80, + .protocol = IPPROTO_TCP, + .in_pkts = 8, + .out_pkts = 8, + .app = "Google", + .app_proto = "HTTP", + .app_type = "Web", + .if_name = dp_test_intf_real("dp2T1", real_ifname), + }; + dp_test_verify_log_file(&log_result, 1); + + /* + * End + */ + + free(ins_pre); + free(ins_post); + free(outs_pre); + free(outs_post); + + /************************************************************* + * Cleanup + *************************************************************/ + + dp_test_create_and_send_cfg_feat_msg( + false, dp_test_intf_real("dp2T1", real_ifname)); + + dp_test_npf_cmd_fmt(false, + "npf-ut delete app-firewall:ALLOWED-SITES 10000"); + dp_test_npf_cmd_fmt(false, "npf-ut delete app-firewall:ALLOWED-SITES"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, + "npf-ut detach interface:dpT21 fw-out fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, "npf-ut delete fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + + dp_test_npf_clear_sessions(); + + dp_test_netlink_del_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_del_neigh("dp2T1", "216.58.200.78", "aa:bb:cc:18:0:1"); + + dp_test_nl_del_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_del_ip_addr_and_connected("dp2T1", "216.58.200.1/24"); +} DP_END_TEST; + +/* + * 100.101.102.x 216.58.200.x + * +----------+ + * .1| |.1 + * .103 -------------------| UUT |------------------ .78 (google.com) + * dp1T0| |dp2T1 + * +----------+ + * + * + * http get google.com expired session (with DPI) + */ +DP_START_TEST(logflowdpi, google_expired) +{ + char *dp1T0_mac = dp_test_intf_name2mac_str("dp1T0"); + char *dp2T1_mac = dp_test_intf_name2mac_str("dp2T1"); + + /* Setup interfaces and neighbours */ + dp_test_nl_add_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_add_ip_addr_and_connected("dp2T1", "216.58.200.1/24"); + + dp_test_netlink_add_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_add_neigh("dp2T1", "216.58.200.78", + "aa:bb:cc:18:0:1"); + + _dp_test_remove_log_file(); + + /* Enable the feature */ + char real_ifname[IFNAMSIZ]; + dp_test_create_and_send_cfg_feat_msg( + true, dp_test_intf_real("dp2T1", real_ifname)); + + struct dp_test_pkt_desc_t *ins_pre, *ins_post; + struct dp_test_pkt_desc_t *outs_pre, *outs_post; + + ins_pre = dpt_pdesc_v4_create( + "Inside pre", IPPROTO_TCP, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + dp1T0_mac, "216.58.200.78", 80, + "dp1T0", "dp2T1"); + + ins_post = dpt_pdesc_v4_create( + "Inside post", IPPROTO_TCP, + dp2T1_mac, "100.101.102.103", 49152, + "aa:bb:cc:18:0:1", "216.58.200.78", 80, + "dp1T0", "dp2T1"); + + outs_pre = dpt_pdesc_v4_create( + "Outside pre", IPPROTO_TCP, + "aa:bb:cc:18:0:1", "216.58.200.78", 80, + dp2T1_mac, "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + outs_post = dpt_pdesc_v4_create( + "Outside post", IPPROTO_TCP, + dp1T0_mac, "216.58.200.78", 80, + "aa:bb:cc:16:0:20", "100.101.102.103", 49152, + "dp2T1", "dp1T0"); + + /* + * dpi app firewall + */ + dp_test_npf_cmd_fmt(false, + "npf-ut delete app-firewall:ALLOWED-SITES 10000"); + dp_test_npf_cmd_fmt(false, "npf-ut add app-firewall:ALLOWED-SITES 100 " + "action=accept engine=ndpi protocol=http"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, "npf-ut add fw:DPI 200 action=accept " + "proto-final=6 stateful=y dst-port=80 " + "rproc=app-firewall(ALLOWED-SITES)"); + dp_test_npf_cmd_fmt(false, + "npf-ut add fw:DPI 1000 action=accept stateful=y"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, + "npf-ut attach interface:dpT21 fw-out fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + + struct dpt_tcp_flow tcp_call = { + .text[0] = '\0', + .isn = {0, 0}, + .desc[DPT_FORW] = { + .pre = ins_pre, + .pst = ins_post, + }, + .desc[DPT_BACK] = { + .pre = outs_pre, + .pst = outs_post, + }, + .test_cb = NULL, + .post_cb = NULL, + }; + + /* Expected LAST-ACK state */ + dpt_tcp_call(&tcp_call, http_google_pkt, + ARRAY_SIZE(http_google_pkt) - 1, + 0, 0, NULL, 0); + + /* Simulate session expired */ + dp_test_session_gc(); + + /* Simulate export log */ + export_log(); + + struct dp_test_flow_log_result log_result = { + .src = "100.101.102.103", + .src_port = 49152, + .dst = "216.58.200.78", + .dst_port = 80, + .protocol = IPPROTO_TCP, + .in_pkts = 7, + .out_pkts = 8, + .app = "Google", + .app_proto = "HTTP", + .app_type = "Web", + .if_name = dp_test_intf_real("dp2T1", real_ifname), + }; + dp_test_verify_log_file(&log_result, 1); + + /* + * End + */ + + free(ins_pre); + free(ins_post); + free(outs_pre); + free(outs_post); + + /************************************************************* + * Cleanup + *************************************************************/ + + dp_test_create_and_send_cfg_feat_msg( + false, dp_test_intf_real("dp2T1", real_ifname)); + + dp_test_npf_cmd_fmt(false, + "npf-ut delete app-firewall:ALLOWED-SITES 10000"); + dp_test_npf_cmd_fmt(false, "npf-ut delete app-firewall:ALLOWED-SITES"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, + "npf-ut detach interface:dpT21 fw-out fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + dp_test_npf_cmd_fmt(false, "npf-ut delete fw:DPI"); + dp_test_npf_cmd_fmt(false, "npf-ut commit"); + + dp_test_npf_clear_sessions(); + + dp_test_netlink_del_neigh("dp1T0", "100.101.102.103", + "aa:bb:cc:16:0:20"); + dp_test_netlink_del_neigh("dp2T1", "216.58.200.78", + "aa:bb:cc:18:0:1"); + + dp_test_nl_del_ip_addr_and_connected("dp1T0", "100.101.102.1/24"); + dp_test_nl_del_ip_addr_and_connected("dp2T1", "216.58.200.1/24"); + +} DP_END_TEST; + +static const char *plugin_name = "dp_test_pipeline_flowstat"; + +int dp_ut_plugin_init(const char **name) +{ + int rv = 0; + + *name = plugin_name; + + return rv; +} diff --git a/tests/whole_dp/src/pcap/gen_pkt.py b/tests/whole_dp/src/pcap/gen_pkt.py new file mode 100755 index 00000000..d23c4416 --- /dev/null +++ b/tests/whole_dp/src/pcap/gen_pkt.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, SafePoint. All rights reserved. +# +# SPDX-License-Identifier: LGPL-2.1-only +# + +import pcapkit + + +def extract(file): + src = None + items = [] + extraction = pcapkit.extract(fin=file, nofile=True) + for frame in extraction.frame: + if frame.info.protocols.startswith('Ethernet:IPv4:TCP'): + ethernet = frame.payload + ipv4 = ethernet.payload + tcp = ipv4.payload + tcp_len = len(tcp.payload.data) + + if src is None: + src = str(ipv4.src) + + dp_dir = 'DPT_FORW' if str(ipv4.src) == src else 'DPT_BACK' + c_data = ''.join(f'\\x{i:02x}' for i in tcp.payload.data) + c_data = f'"{c_data}"' if tcp_len else 'NULL' + + # flags + m = { + 'urg': 'urg', + 'ack': 'ack', + 'psh': 'push', + 'rst': 'rst', + 'syn': 'syn', + 'fin': 'fin', + } + flags = [m[i] for i in list(tcp.info.flags) if getattr(tcp.info.flags, i, False)] + flags = sorted(flags, key=lambda f: f == 'ack') + flags = ' | '.join(f'TH_{i}'.upper() for i in flags) + + # fix fin,ack + if 'TH_FIN' in flags and 'TH_ACK' in flags: + for f in ['TH_FIN', 'TH_ACK']: + val = f'{dp_dir}, {f}, 0, NULL, 0, NULL' + items.append(val) + else: + val = f'{dp_dir}, {flags}, {tcp_len}, {c_data}, 0, NULL' + items.append(val) + + name = file.rsplit('.', 1)[0].replace('-', '_') + '_pkt' + result = [] + result += [f'/* generated from: {file} */'] + result += [''] + result += ['#include "dp_test_lib_tcp.h"'] + result += [''] + result += ['struct dpt_tcp_flow_pkt %s[] = {' % name] + for i in items: + result += ['\t{%s},' % i] + result += ['};'] + result += [''] + result = '\n'.join(result) + return result + + +def _main(): + import argparse + + parser = argparse.ArgumentParser('Flow test generator') + parser.add_argument('file', help='file.pcap') + args = parser.parse_args() + + result = extract(args.file) + + out_file = args.file.rsplit('.', 1)[0] + '.h' + with open(out_file, 'w') as f: + f.write(result) + print(f'Wrote to file {out_file}') + + +if __name__ == '__main__': + _main() diff --git a/tests/whole_dp/src/pcap/http-example.h b/tests/whole_dp/src/pcap/http-example.h new file mode 100644 index 00000000..edbcb4b7 --- /dev/null +++ b/tests/whole_dp/src/pcap/http-example.h @@ -0,0 +1,126 @@ +/* generated from: http-example.pcap */ + +#include "dp_test_lib_tcp.h" + +struct dpt_tcp_flow_pkt http_example_pkt[] = { + {DPT_FORW, TH_SYN, 0, NULL, 0, NULL}, + {DPT_BACK, TH_SYN | TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_PUSH | TH_ACK, 75, + "\x47\x45\x54\x20\x2f\x20\x48\x54\x54\x50\x2f\x31\x2e\x31\x0d\x0a\x48" + "\x6f\x73\x74\x3a\x20\x65\x78\x61\x6d\x70\x6c\x65\x2e\x63\x6f\x6d\x0d" + "\x0a\x55\x73\x65\x72\x2d\x41\x67\x65\x6e\x74\x3a\x20\x63\x75\x72\x6c" + "\x2f\x37\x2e\x34\x37\x2e\x30\x0d\x0a\x41\x63\x63\x65\x70\x74\x3a\x20" + "\x2a\x2f\x2a\x0d\x0a\x0d\x0a", + 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_PUSH | TH_ACK, 17, + "\x48\x54\x54\x50\x2f\x31\x2e\x31\x20\x32\x30\x30\x20\x4f\x4b\x0d\x0a", + 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_PUSH | TH_ACK, 337, + "\x41\x67\x65\x3a\x20\x35\x36\x35\x37\x35\x37\x0d\x0a\x43\x61\x63\x68" + "\x65\x2d\x43\x6f\x6e\x74\x72\x6f\x6c\x3a\x20\x6d\x61\x78\x2d\x61\x67" + "\x65\x3d\x36\x30\x34\x38\x30\x30\x0d\x0a\x43\x6f\x6e\x74\x65\x6e\x74" + "\x2d\x54\x79\x70\x65\x3a\x20\x74\x65\x78\x74\x2f\x68\x74\x6d\x6c\x3b" + "\x20\x63\x68\x61\x72\x73\x65\x74\x3d\x55\x54\x46\x2d\x38\x0d\x0a\x44" + "\x61\x74\x65\x3a\x20\x53\x61\x74\x2c\x20\x30\x33\x20\x41\x70\x72\x20" + "\x32\x30\x32\x31\x20\x30\x35\x3a\x35\x31\x3a\x32\x39\x20\x47\x4d\x54" + "\x0d\x0a\x45\x74\x61\x67\x3a\x20\x22\x33\x31\x34\x37\x35\x32\x36\x39" + "\x34\x37\x2b\x69\x64\x65\x6e\x74\x22\x0d\x0a\x45\x78\x70\x69\x72\x65" + "\x73\x3a\x20\x53\x61\x74\x2c\x20\x31\x30\x20\x41\x70\x72\x20\x32\x30" + "\x32\x31\x20\x30\x35\x3a\x35\x31\x3a\x32\x39\x20\x47\x4d\x54\x0d\x0a" + "\x4c\x61\x73\x74\x2d\x4d\x6f\x64\x69\x66\x69\x65\x64\x3a\x20\x54\x68" + "\x75\x2c\x20\x31\x37\x20\x4f\x63\x74\x20\x32\x30\x31\x39\x20\x30\x37" + "\x3a\x31\x38\x3a\x32\x36\x20\x47\x4d\x54\x0d\x0a\x53\x65\x72\x76\x65" + "\x72\x3a\x20\x45\x43\x53\x20\x28\x6f\x78\x72\x2f\x38\x33\x32\x34\x29" + "\x0d\x0a\x56\x61\x72\x79\x3a\x20\x41\x63\x63\x65\x70\x74\x2d\x45\x6e" + "\x63\x6f\x64\x69\x6e\x67\x0d\x0a\x58\x2d\x43\x61\x63\x68\x65\x3a\x20" + "\x48\x49\x54\x0d\x0a\x43\x6f\x6e\x74\x65\x6e\x74\x2d\x4c\x65\x6e\x67" + "\x74\x68\x3a\x20\x31\x32\x35\x36\x0d\x0a\x43\x6f\x6e\x6e\x65\x63\x74" + "\x69\x6f\x6e\x3a\x20\x63\x6c\x6f\x73\x65\x0d\x0a\x0d\x0a", + 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_PUSH | TH_ACK, 1256, + "\x3c\x21\x64\x6f\x63\x74\x79\x70\x65\x20\x68\x74\x6d\x6c\x3e\x0a\x3c" + "\x68\x74\x6d\x6c\x3e\x0a\x3c\x68\x65\x61\x64\x3e\x0a\x20\x20\x20\x20" + "\x3c\x74\x69\x74\x6c\x65\x3e\x45\x78\x61\x6d\x70\x6c\x65\x20\x44\x6f" + "\x6d\x61\x69\x6e\x3c\x2f\x74\x69\x74\x6c\x65\x3e\x0a\x0a\x20\x20\x20" + "\x20\x3c\x6d\x65\x74\x61\x20\x63\x68\x61\x72\x73\x65\x74\x3d\x22\x75" + "\x74\x66\x2d\x38\x22\x20\x2f\x3e\x0a\x20\x20\x20\x20\x3c\x6d\x65\x74" + "\x61\x20\x68\x74\x74\x70\x2d\x65\x71\x75\x69\x76\x3d\x22\x43\x6f\x6e" + "\x74\x65\x6e\x74\x2d\x74\x79\x70\x65\x22\x20\x63\x6f\x6e\x74\x65\x6e" + "\x74\x3d\x22\x74\x65\x78\x74\x2f\x68\x74\x6d\x6c\x3b\x20\x63\x68\x61" + "\x72\x73\x65\x74\x3d\x75\x74\x66\x2d\x38\x22\x20\x2f\x3e\x0a\x20\x20" + "\x20\x20\x3c\x6d\x65\x74\x61\x20\x6e\x61\x6d\x65\x3d\x22\x76\x69\x65" + "\x77\x70\x6f\x72\x74\x22\x20\x63\x6f\x6e\x74\x65\x6e\x74\x3d\x22\x77" + "\x69\x64\x74\x68\x3d\x64\x65\x76\x69\x63\x65\x2d\x77\x69\x64\x74\x68" + "\x2c\x20\x69\x6e\x69\x74\x69\x61\x6c\x2d\x73\x63\x61\x6c\x65\x3d\x31" + "\x22\x20\x2f\x3e\x0a\x20\x20\x20\x20\x3c\x73\x74\x79\x6c\x65\x20\x74" + "\x79\x70\x65\x3d\x22\x74\x65\x78\x74\x2f\x63\x73\x73\x22\x3e\x0a\x20" + "\x20\x20\x20\x62\x6f\x64\x79\x20\x7b\x0a\x20\x20\x20\x20\x20\x20\x20" + "\x20\x62\x61\x63\x6b\x67\x72\x6f\x75\x6e\x64\x2d\x63\x6f\x6c\x6f\x72" + "\x3a\x20\x23\x66\x30\x66\x30\x66\x32\x3b\x0a\x20\x20\x20\x20\x20\x20" + "\x20\x20\x6d\x61\x72\x67\x69\x6e\x3a\x20\x30\x3b\x0a\x20\x20\x20\x20" + "\x20\x20\x20\x20\x70\x61\x64\x64\x69\x6e\x67\x3a\x20\x30\x3b\x0a\x20" + "\x20\x20\x20\x20\x20\x20\x20\x66\x6f\x6e\x74\x2d\x66\x61\x6d\x69\x6c" + "\x79\x3a\x20\x2d\x61\x70\x70\x6c\x65\x2d\x73\x79\x73\x74\x65\x6d\x2c" + "\x20\x73\x79\x73\x74\x65\x6d\x2d\x75\x69\x2c\x20\x42\x6c\x69\x6e\x6b" + "\x4d\x61\x63\x53\x79\x73\x74\x65\x6d\x46\x6f\x6e\x74\x2c\x20\x22\x53" + "\x65\x67\x6f\x65\x20\x55\x49\x22\x2c\x20\x22\x4f\x70\x65\x6e\x20\x53" + "\x61\x6e\x73\x22\x2c\x20\x22\x48\x65\x6c\x76\x65\x74\x69\x63\x61\x20" + "\x4e\x65\x75\x65\x22\x2c\x20\x48\x65\x6c\x76\x65\x74\x69\x63\x61\x2c" + "\x20\x41\x72\x69\x61\x6c\x2c\x20\x73\x61\x6e\x73\x2d\x73\x65\x72\x69" + "\x66\x3b\x0a\x20\x20\x20\x20\x20\x20\x20\x20\x0a\x20\x20\x20\x20\x7d" + "\x0a\x20\x20\x20\x20\x64\x69\x76\x20\x7b\x0a\x20\x20\x20\x20\x20\x20" + "\x20\x20\x77\x69\x64\x74\x68\x3a\x20\x36\x30\x30\x70\x78\x3b\x0a\x20" + "\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x72\x67\x69\x6e\x3a\x20\x35\x65" + "\x6d\x20\x61\x75\x74\x6f\x3b\x0a\x20\x20\x20\x20\x20\x20\x20\x20\x70" + "\x61\x64\x64\x69\x6e\x67\x3a\x20\x32\x65\x6d\x3b\x0a\x20\x20\x20\x20" + "\x20\x20\x20\x20\x62\x61\x63\x6b\x67\x72\x6f\x75\x6e\x64\x2d\x63\x6f" + "\x6c\x6f\x72\x3a\x20\x23\x66\x64\x66\x64\x66\x66\x3b\x0a\x20\x20\x20" + "\x20\x20\x20\x20\x20\x62\x6f\x72\x64\x65\x72\x2d\x72\x61\x64\x69\x75" + "\x73\x3a\x20\x30\x2e\x35\x65\x6d\x3b\x0a\x20\x20\x20\x20\x20\x20\x20" + "\x20\x62\x6f\x78\x2d\x73\x68\x61\x64\x6f\x77\x3a\x20\x32\x70\x78\x20" + "\x33\x70\x78\x20\x37\x70\x78\x20\x32\x70\x78\x20\x72\x67\x62\x61\x28" + "\x30\x2c\x30\x2c\x30\x2c\x30\x2e\x30\x32\x29\x3b\x0a\x20\x20\x20\x20" + "\x7d\x0a\x20\x20\x20\x20\x61\x3a\x6c\x69\x6e\x6b\x2c\x20\x61\x3a\x76" + "\x69\x73\x69\x74\x65\x64\x20\x7b\x0a\x20\x20\x20\x20\x20\x20\x20\x20" + "\x63\x6f\x6c\x6f\x72\x3a\x20\x23\x33\x38\x34\x38\x38\x66\x3b\x0a\x20" + "\x20\x20\x20\x20\x20\x20\x20\x74\x65\x78\x74\x2d\x64\x65\x63\x6f\x72" + "\x61\x74\x69\x6f\x6e\x3a\x20\x6e\x6f\x6e\x65\x3b\x0a\x20\x20\x20\x20" + "\x7d\x0a\x20\x20\x20\x20\x40\x6d\x65\x64\x69\x61\x20\x28\x6d\x61\x78" + "\x2d\x77\x69\x64\x74\x68\x3a\x20\x37\x30\x30\x70\x78\x29\x20\x7b\x0a" + "\x20\x20\x20\x20\x20\x20\x20\x20\x64\x69\x76\x20\x7b\x0a\x20\x20\x20" + "\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x72\x67\x69\x6e\x3a\x20" + "\x30\x20\x61\x75\x74\x6f\x3b\x0a\x20\x20\x20\x20\x20\x20\x20\x20\x20" + "\x20\x20\x20\x77\x69\x64\x74\x68\x3a\x20\x61\x75\x74\x6f\x3b\x0a\x20" + "\x20\x20\x20\x20\x20\x20\x20\x7d\x0a\x20\x20\x20\x20\x7d\x0a\x20\x20" + "\x20\x20\x3c\x2f\x73\x74\x79\x6c\x65\x3e\x20\x20\x20\x20\x0a\x3c\x2f" + "\x68\x65\x61\x64\x3e\x0a\x0a\x3c\x62\x6f\x64\x79\x3e\x0a\x3c\x64\x69" + "\x76\x3e\x0a\x20\x20\x20\x20\x3c\x68\x31\x3e\x45\x78\x61\x6d\x70\x6c" + "\x65\x20\x44\x6f\x6d\x61\x69\x6e\x3c\x2f\x68\x31\x3e\x0a\x20\x20\x20" + "\x20\x3c\x70\x3e\x54\x68\x69\x73\x20\x64\x6f\x6d\x61\x69\x6e\x20\x69" + "\x73\x20\x66\x6f\x72\x20\x75\x73\x65\x20\x69\x6e\x20\x69\x6c\x6c\x75" + "\x73\x74\x72\x61\x74\x69\x76\x65\x20\x65\x78\x61\x6d\x70\x6c\x65\x73" + "\x20\x69\x6e\x20\x64\x6f\x63\x75\x6d\x65\x6e\x74\x73\x2e\x20\x59\x6f" + "\x75\x20\x6d\x61\x79\x20\x75\x73\x65\x20\x74\x68\x69\x73\x0a\x20\x20" + "\x20\x20\x64\x6f\x6d\x61\x69\x6e\x20\x69\x6e\x20\x6c\x69\x74\x65\x72" + "\x61\x74\x75\x72\x65\x20\x77\x69\x74\x68\x6f\x75\x74\x20\x70\x72\x69" + "\x6f\x72\x20\x63\x6f\x6f\x72\x64\x69\x6e\x61\x74\x69\x6f\x6e\x20\x6f" + "\x72\x20\x61\x73\x6b\x69\x6e\x67\x20\x66\x6f\x72\x20\x70\x65\x72\x6d" + "\x69\x73\x73\x69\x6f\x6e\x2e\x3c\x2f\x70\x3e\x0a\x20\x20\x20\x20\x3c" + "\x70\x3e\x3c\x61\x20\x68\x72\x65\x66\x3d\x22\x68\x74\x74\x70\x73\x3a" + "\x2f\x2f\x77\x77\x77\x2e\x69\x61\x6e\x61\x2e\x6f\x72\x67\x2f\x64\x6f" + "\x6d\x61\x69\x6e\x73\x2f\x65\x78\x61\x6d\x70\x6c\x65\x22\x3e\x4d\x6f" + "\x72\x65\x20\x69\x6e\x66\x6f\x72\x6d\x61\x74\x69\x6f\x6e\x2e\x2e\x2e" + "\x3c\x2f\x61\x3e\x3c\x2f\x70\x3e\x0a\x3c\x2f\x64\x69\x76\x3e\x0a\x3c" + "\x2f\x62\x6f\x64\x79\x3e\x0a\x3c\x2f\x68\x74\x6d\x6c\x3e\x0a", + 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_FIN, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_FIN, 0, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, +}; diff --git a/tests/whole_dp/src/pcap/http-example.pcap b/tests/whole_dp/src/pcap/http-example.pcap new file mode 100644 index 00000000..b2c231e4 Binary files /dev/null and b/tests/whole_dp/src/pcap/http-example.pcap differ diff --git a/tests/whole_dp/src/pcap/http-google.h b/tests/whole_dp/src/pcap/http-google.h new file mode 100644 index 00000000..84e8a9ae --- /dev/null +++ b/tests/whole_dp/src/pcap/http-google.h @@ -0,0 +1,64 @@ +/* generated from: http-google.pcap */ + +#include "dp_test_lib_tcp.h" + +struct dpt_tcp_flow_pkt http_google_pkt[] = { + {DPT_FORW, TH_SYN, 0, NULL, 0, NULL}, + {DPT_BACK, TH_SYN | TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_PUSH | TH_ACK, 74, + "\x47\x45\x54\x20\x2f\x20\x48\x54\x54\x50\x2f\x31\x2e\x31\x0d\x0a\x48" + "\x6f\x73\x74\x3a\x20\x67\x6f\x6f\x67\x6c\x65\x2e\x63\x6f\x6d\x0d\x0a" + "\x55\x73\x65\x72\x2d\x41\x67\x65\x6e\x74\x3a\x20\x63\x75\x72\x6c\x2f" + "\x37\x2e\x34\x37\x2e\x30\x0d\x0a\x41\x63\x63\x65\x70\x74\x3a\x20\x2a" + "\x2f\x2a\x0d\x0a\x0d\x0a", + 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_PUSH | TH_ACK, 32, + "\x48\x54\x54\x50\x2f\x31\x2e\x31\x20\x33\x30\x31\x20\x4d\x6f\x76\x65" + "\x64\x20\x50\x65\x72\x6d\x61\x6e\x65\x6e\x74\x6c\x79\x0d\x0a", + 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_PUSH | TH_ACK, 296, + "\x4c\x6f\x63\x61\x74\x69\x6f\x6e\x3a\x20\x68\x74\x74\x70\x3a\x2f\x2f" + "\x77\x77\x77\x2e\x67\x6f\x6f\x67\x6c\x65\x2e\x63\x6f\x6d\x2f\x0d\x0a" + "\x43\x6f\x6e\x74\x65\x6e\x74\x2d\x54\x79\x70\x65\x3a\x20\x74\x65\x78" + "\x74\x2f\x68\x74\x6d\x6c\x3b\x20\x63\x68\x61\x72\x73\x65\x74\x3d\x55" + "\x54\x46\x2d\x38\x0d\x0a\x44\x61\x74\x65\x3a\x20\x57\x65\x64\x2c\x20" + "\x33\x31\x20\x4d\x61\x72\x20\x32\x30\x32\x31\x20\x30\x37\x3a\x34\x32" + "\x3a\x34\x36\x20\x47\x4d\x54\x0d\x0a\x45\x78\x70\x69\x72\x65\x73\x3a" + "\x20\x46\x72\x69\x2c\x20\x33\x30\x20\x41\x70\x72\x20\x32\x30\x32\x31" + "\x20\x30\x37\x3a\x34\x32\x3a\x34\x36\x20\x47\x4d\x54\x0d\x0a\x43\x61" + "\x63\x68\x65\x2d\x43\x6f\x6e\x74\x72\x6f\x6c\x3a\x20\x70\x75\x62\x6c" + "\x69\x63\x2c\x20\x6d\x61\x78\x2d\x61\x67\x65\x3d\x32\x35\x39\x32\x30" + "\x30\x30\x0d\x0a\x53\x65\x72\x76\x65\x72\x3a\x20\x67\x77\x73\x0d\x0a" + "\x43\x6f\x6e\x74\x65\x6e\x74\x2d\x4c\x65\x6e\x67\x74\x68\x3a\x20\x32" + "\x31\x39\x0d\x0a\x58\x2d\x58\x53\x53\x2d\x50\x72\x6f\x74\x65\x63\x74" + "\x69\x6f\x6e\x3a\x20\x30\x0d\x0a\x58\x2d\x46\x72\x61\x6d\x65\x2d\x4f" + "\x70\x74\x69\x6f\x6e\x73\x3a\x20\x53\x41\x4d\x45\x4f\x52\x49\x47\x49" + "\x4e\x0d\x0a\x43\x6f\x6e\x6e\x65\x63\x74\x69\x6f\x6e\x3a\x20\x63\x6c" + "\x6f\x73\x65\x0d\x0a\x0d\x0a", + 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_PUSH | TH_ACK, 219, + "\x3c\x48\x54\x4d\x4c\x3e\x3c\x48\x45\x41\x44\x3e\x3c\x6d\x65\x74\x61" + "\x20\x68\x74\x74\x70\x2d\x65\x71\x75\x69\x76\x3d\x22\x63\x6f\x6e\x74" + "\x65\x6e\x74\x2d\x74\x79\x70\x65\x22\x20\x63\x6f\x6e\x74\x65\x6e\x74" + "\x3d\x22\x74\x65\x78\x74\x2f\x68\x74\x6d\x6c\x3b\x63\x68\x61\x72\x73" + "\x65\x74\x3d\x75\x74\x66\x2d\x38\x22\x3e\x0a\x3c\x54\x49\x54\x4c\x45" + "\x3e\x33\x30\x31\x20\x4d\x6f\x76\x65\x64\x3c\x2f\x54\x49\x54\x4c\x45" + "\x3e\x3c\x2f\x48\x45\x41\x44\x3e\x3c\x42\x4f\x44\x59\x3e\x0a\x3c\x48" + "\x31\x3e\x33\x30\x31\x20\x4d\x6f\x76\x65\x64\x3c\x2f\x48\x31\x3e\x0a" + "\x54\x68\x65\x20\x64\x6f\x63\x75\x6d\x65\x6e\x74\x20\x68\x61\x73\x20" + "\x6d\x6f\x76\x65\x64\x0a\x3c\x41\x20\x48\x52\x45\x46\x3d\x22\x68\x74" + "\x74\x70\x3a\x2f\x2f\x77\x77\x77\x2e\x67\x6f\x6f\x67\x6c\x65\x2e\x63" + "\x6f\x6d\x2f\x22\x3e\x68\x65\x72\x65\x3c\x2f\x41\x3e\x2e\x0d\x0a\x3c" + "\x2f\x42\x4f\x44\x59\x3e\x3c\x2f\x48\x54\x4d\x4c\x3e\x0d\x0a", + 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_FIN, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, + {DPT_FORW, TH_FIN, 0, NULL, 0, NULL}, + {DPT_FORW, TH_ACK, 0, NULL, 0, NULL}, + {DPT_BACK, TH_ACK, 0, NULL, 0, NULL}, +}; diff --git a/tests/whole_dp/src/pcap/http-google.pcap b/tests/whole_dp/src/pcap/http-google.pcap new file mode 100644 index 00000000..8f4d4fc4 Binary files /dev/null and b/tests/whole_dp/src/pcap/http-google.pcap differ