From 927f7ae2dda6e960102ee67acfb28eb0800321cc Mon Sep 17 00:00:00 2001 From: Evan Zelkowitz Date: Fri, 19 Jun 2020 23:54:12 +0000 Subject: [PATCH 1/5] Add maxmind acl plugin This plugin is similar to the current geoip acl plugin however it is based on the libmaxminddb library. The GeoIP library is considered legacy at this point and it only supports the old dat formatted databases. This will work with the newer more common mmdb format. It takes a yaml formatted configuration file as shown in the docs, which specifies the database to use for that remap, and then any sets of allow and deny rules. Currently the only rules supported are `country`,`ip`, and `regex`. Deny rules will always take precedence if both allow and deny rules are set, and in the case of having both sets of rules or only allow rules the default action is to deny a connection. However if no accept rules are specified then the default action will flip to always allow, since it doesnt make much sense to leave it as blocking everything if you only want to deny. In that case all connections are allowed except those that fall into one of the deny rules. --- configure.ac | 15 + doc/admin-guide/plugins/index.en.rst | 4 + doc/admin-guide/plugins/maxmind_acl.en.rst | 78 +++ plugins/Makefile.am | 5 + plugins/experimental/maxmind_acl/Makefile.inc | 28 + .../experimental/maxmind_acl/maxmind_acl.cc | 117 ++++ plugins/experimental/maxmind_acl/mmdb.cc | 570 ++++++++++++++++++ plugins/experimental/maxmind_acl/mmdb.h | 120 ++++ 8 files changed, 937 insertions(+) create mode 100644 doc/admin-guide/plugins/maxmind_acl.en.rst create mode 100644 plugins/experimental/maxmind_acl/Makefile.inc create mode 100644 plugins/experimental/maxmind_acl/maxmind_acl.cc create mode 100644 plugins/experimental/maxmind_acl/mmdb.cc create mode 100644 plugins/experimental/maxmind_acl/mmdb.h diff --git a/configure.ac b/configure.ac index f00e40e0ea9..336a9ee1a8a 100644 --- a/configure.ac +++ b/configure.ac @@ -1685,6 +1685,21 @@ AC_CHECK_HEADERS([GeoIP.h], [ ]) ]) +# +# Check for libmaxmind. This is the maxmind v2 API where GeoIP is the legacy +# v1 dat file based API +# +AC_CHECK_HEADERS([maxminddb.h], [ + AC_CHECK_LIB([maxminddb], [MMDB_open], [ + AC_SUBST([MAXMINDDB_LIBS], ["-lmaxminddb"]) + AC_SUBST(has_maxmind, 1) + ], [ + AC_SUBST([MAXMINDDB_LIBS], [""]) + AC_SUBST(has_maxmind, 0) + ]) +]) + +AM_CONDITIONAL([BUILD_MAXMIND_ACL_PLUGIN], [test "x${has_maxmind}" = "x1" ]) # Right now, the healthcheck plugins requires inotify_init (and friends) AM_CONDITIONAL([BUILD_HEALTHCHECK_PLUGIN], [ test "$ac_cv_func_inotify_init" = "yes" ]) diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index 48a3d5bdf64..f247182e0e4 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -155,6 +155,7 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Header Frequency Hook Trace JA3 Fingerprint + Maxmind ACL Memcache Metalink Money Trace @@ -195,6 +196,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`JA3 Fingerprint ` Calculates JA3 Fingerprints for incoming SSL traffic. +:doc:`MaxMind ACL ` + ACL based on the maxmind geo databases (GeoIP2 mmdb and libmaxminddb) + :doc:`Memcache ` Implements the memcache protocol for cache contents. diff --git a/doc/admin-guide/plugins/maxmind_acl.en.rst b/doc/admin-guide/plugins/maxmind_acl.en.rst new file mode 100644 index 00000000000..e783b0bec1e --- /dev/null +++ b/doc/admin-guide/plugins/maxmind_acl.en.rst @@ -0,0 +1,78 @@ +.. _admin-plugins-maxmind-acl: + +MaxMind ACL Plugin +****************** + +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +This remap plugin provides allow and deny functionality based on the libmaxminddb +library and GeoIP2 databases (mmdb format). It requires libmaxminddb to run +and the associated development headers in order to build. You can find a sample +mmdb-lite database on the maxmind website or provide your own. You must provide a database +for any usages and specify it in the configuration file as shown below. + +Configuration +============= + +The plugin takes a single pparam which is the location of the configuration yaml +file. This can either be relative to the ATS configuration directory or an absolute path :: + + map http://example.com/music http://music.example.com @plugin=maxmind_acl.so @pparam=maxmind.yaml + +An example configuration :: + + maxmind: + database: GeoIP2-City.mmdb + html: deny.html + allow: + country: + - US + ip: + - 127.0.0.1 + - 192.168.10.0/20 + deny: + country: + - DE + ip: + - 127.0.0.1 + regex: + - [US, ".*\\.txt"] # Because these get parsed you must escape the escape of the ``.`` in order to have it be escaped in the regex, resulting in ".*\.txt" + - [US, ".*\\.mp3"] + +Rules +===== + +You can mix and match the allow rules and deny rules, however deny rules will always take precedence so in the above case ``127.0.0.1`` would be denied. +The IP rules can take either single IPs or cidr formatted rules. It will also accept IPv6 IP and ranges. + +The regex portion can be added to both the allow and deny sections for creating allowable or denyable regexes. Each regex takes a country code first and a regex second. +In the above example all requests from the US would be allowed except for those on ``txt`` and ``mp3`` files. More rules should be added as pairs, not as additions to existing lists. + +Currently the only rules available are ``country``, ``ip``, and ``regex``, though more can easily be added if needed. Each config file does require a top level +``maxmind`` entry as well as a ``database`` entry for the IP lookups. You can supply a separate database for each remap used in case you use custom +ones and have specific needs per remap. + +One other thing to note. You can reverse the logic of the plugin, so that it will default to always allowing if you do not supply any ``allow`` rules. +In the case you supply no allow rules all connections will be allowed through except those that fall in to any of the deny rule lists. In the above example +the rule of denying ``DE`` would be a noop because there are allow rules set, so by default everything is blocked unless it is explicitly in an allow rule. +However in this case the regexes would still apply since they are based on an allowable country. + +Optional +======== + +There is an optional ``html`` field which takes a html file that will be used as the body of the response for any denied requests if you wish to use a custom one. diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 1ce513c7e59..03d6dd33600 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -68,6 +68,11 @@ include experimental/geoip_acl/Makefile.inc include experimental/header_freq/Makefile.inc include experimental/hook-trace/Makefile.inc include experimental/inliner/Makefile.inc + +if BUILD_MAXMIND_ACL_PLUGIN +include experimental/maxmind_acl/Makefile.inc +endif + include experimental/memcache/Makefile.inc include experimental/metalink/Makefile.inc include experimental/money_trace/Makefile.inc diff --git a/plugins/experimental/maxmind_acl/Makefile.inc b/plugins/experimental/maxmind_acl/Makefile.inc new file mode 100644 index 00000000000..5a613b5f77f --- /dev/null +++ b/plugins/experimental/maxmind_acl/Makefile.inc @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/maxmind_acl/maxmind_acl.la + +experimental_maxmind_acl_maxmind_acl_la_SOURCES = \ + experimental/maxmind_acl/maxmind_acl.cc \ + experimental/maxmind_acl/mmdb.cc + +experimental_maxmind_acl_maxmind_acl_la_LIBADD = $(MAXMINDDB_LIBS) + +experimental_maxmind_acl_maxmind_acl_la_LDFLAGS = \ + $(AM_LDFLAGS) + +AM_CPPFLAGS += @YAMLCPP_INCLUDES@ diff --git a/plugins/experimental/maxmind_acl/maxmind_acl.cc b/plugins/experimental/maxmind_acl/maxmind_acl.cc new file mode 100644 index 00000000000..bb72e72266b --- /dev/null +++ b/plugins/experimental/maxmind_acl/maxmind_acl.cc @@ -0,0 +1,117 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "mmdb.h" + +static int +config_handler(TSCont cont, TSEvent event, void *edata) +{ + TSMutex mutex; + + mutex = TSContMutexGet(cont); + TSMutexLock(mutex); + + TSDebug(PLUGIN_NAME, "In config Handler"); + Acl *a = static_cast(TSContDataGet(cont)); + + // strdup for const string return + char *config = strdup(a->get_state()->config_file.c_str()); + a->init(config); + free(config); + TSMutexUnlock(mutex); + + // Don't reschedule for TS_EVENT_MGMT_UPDATE + if (event == TS_EVENT_TIMEOUT) { + TSContScheduleOnPool(cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); + } + return 0; +} + +/////////////////////////////////////////////////////////////////////////////// +// Initialize the plugin as a remap plugin. +// +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) +{ + if (api_info->size < sizeof(TSRemapInterface)) { + strncpy(errbuf, "[tsremap_init] - Incorrect size of TSRemapInterface structure", errbuf_size - 1); + return TS_ERROR; + } + + if (api_info->tsremap_version < TSREMAP_VERSION) { + snprintf(errbuf, errbuf_size, "[tsremap_init] - Incorrect API version %ld.%ld", api_info->tsremap_version >> 16, + (api_info->tsremap_version & 0xffff)); + return TS_ERROR; + } + + TSDebug(PLUGIN_NAME, "remap plugin is successfully initialized"); + return TS_SUCCESS; +} + +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int /* errbuf_size */) +{ + TSCont config_cont; + + if (argc < 3) { + TSError("[%s] Unable to create remap instance, missing configuration file", PLUGIN_NAME); + return TS_ERROR; + } + + Acl *a = new Acl(); + *ih = static_cast(a); + if (!a->init(argv[2])) { + TSError("[%s] Failed to initialize maxmind with %s", PLUGIN_NAME, argv[2]); + return TS_ERROR; + } + + config_cont = TSContCreate(config_handler, TSMutexCreate()); + TSContDataSet(config_cont, static_cast(a)); + TSMgmtUpdateRegister(config_cont, PLUGIN_NAME); + + TSDebug(PLUGIN_NAME, "created remap instance with configuration %s", argv[2]); + return TS_SUCCESS; +} + +void +TSRemapDeleteInstance(void *ih) +{ + if (nullptr != ih) { + Acl *const a = static_cast(ih); + delete a; + } +} + +/////////////////////////////////////////////////////////////////////////////// +// Main entry point when used as a remap plugin. +// +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) +{ + if (nullptr == ih) { + TSDebug(PLUGIN_NAME, "No ACLs configured"); + } else { + Acl *a = static_cast(ih); + if (!a->eval(rri, rh)) { + TSDebug(PLUGIN_NAME, "denying request"); + TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_FORBIDDEN); + a->send_html(rh); + } + } + return TSREMAP_NO_REMAP; +} diff --git a/plugins/experimental/maxmind_acl/mmdb.cc b/plugins/experimental/maxmind_acl/mmdb.cc new file mode 100644 index 00000000000..8b1b48a0cfc --- /dev/null +++ b/plugins/experimental/maxmind_acl/mmdb.cc @@ -0,0 +1,570 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "mmdb.h" + +/////////////////////////////////////////////////////////////////////////////// +// Load the config file from param +// check for basics +// Clear out any existing data since this may be a reload +bool +Acl::init(char *filename) +{ + std::string configloc; + struct stat s; + bool status = false; + + YAML::Node maxmind; + + if (filename[0] != '/') { + // relative file + configloc = TSConfigDirGet(); + configloc += "/"; + configloc.append(filename); + } else { + configloc.assign(filename); + } + + if (stat(configloc.c_str(), &s) < 0) { + TSDebug(PLUGIN_NAME, "Could not stat %s", configloc.c_str()); + return status; + } + + if ((plugin_state.last_load == 0) || (s.st_mtime > plugin_state.last_load)) { + TSDebug(PLUGIN_NAME, "Loading config %s, lastload: %ld, file time: %ld", configloc.c_str(), plugin_state.last_load, s.st_mtime); + + try { + _config = YAML::LoadFile(configloc.c_str()); + + if (_config.IsNull()) { + TSDebug(PLUGIN_NAME, "Config file not found or unreadable"); + return status; + } + if (!_config["maxmind"]) { + TSDebug(PLUGIN_NAME, "Config file not in maxmind namespace"); + return status; + } + + // Get our root maxmind node + maxmind = _config["maxmind"]; +#if 0 + // Test junk + for (YAML::const_iterator it = maxmind.begin(); it != maxmind.end(); ++it) { + const std::string &name = it->first.as(); + YAML::NodeType::value type = it->second.Type(); + TSDebug(PLUGIN_NAME, "name: %s, value: %d", name.c_str(), type); + } +#endif + } catch (const YAML::Exception &e) { + TSError("YAML::Exception %s when parsing YAML config file %s for maxmind", e.what(), configloc.c_str()); + return status; + } + + // Find our database name and convert to full path as needed + status = loaddb(maxmind["database"]); + + if (!status) { + TSDebug(PLUGIN_NAME, "Failed to load MaxMind Database"); + return status; + } + + // Clear out existing data, these may no longer exist in a new config and so we + // dont want old ones left behind + allow_country.clear(); + allow_ip_map.clear(); + deny_ip_map.clear(); + allow_regex.clear(); + deny_regex.clear(); + _html.clear(); + default_allow = false; + + if (loadallow(maxmind["allow"])) { + TSDebug(PLUGIN_NAME, "Loaded Allow ruleset"); + status = true; + } else { + // We have no proper allow ruleset + // setting to allow by default to only apply deny rules + default_allow = true; + } + + if (loaddeny(maxmind["deny"])) { + TSDebug(PLUGIN_NAME, "Loaded Deny ruleset"); + status = true; + } + + loadhtml(maxmind["html"]); + + if (!status) { + TSDebug(PLUGIN_NAME, "Failed to load any rulesets, none specified"); + status = false; + } + + plugin_state.config_file = configloc; + plugin_state.last_load = time(NULL); + } + + return status; +} + +/////////////////////////////////////////////////////////////////////////////// +// Parse the deny list country codes and IPs +bool +Acl::loaddeny(YAML::Node denyNode) +{ + if (!denyNode) { + TSDebug(PLUGIN_NAME, "No Deny rules set"); + return false; + } + if (denyNode.IsNull()) { + TSDebug(PLUGIN_NAME, "Deny rules are NULL"); + return false; + } + +#if 0 + // Test junk + for (YAML::const_iterator it = denyNode.begin(); it != denyNode.end(); ++it) { + const std::string &name = it->first.as(); + YAML::NodeType::value type = it->second.Type(); + TSDebug(PLUGIN_NAME, "name: %s, value: %d", name.c_str(), type); + } +#endif + + // Load Allowable Country codes + try { + if (denyNode["country"]) { + YAML::Node country = denyNode["country"]; + if (!country.IsNull()) { + if (country.IsSequence()) { + for (std::size_t i = 0; i < country.size(); i++) { + allow_country.insert_or_assign(country[i].as(), false); + } + } else { + TSDebug(PLUGIN_NAME, "Invalid country code allow list yaml"); + } + } + } + } catch (const YAML::Exception &e) { + TSDebug("YAML::Exception %s when parsing YAML config file country code deny list for maxmind", e.what()); + return false; + } + + // Load Denyable IPs + try { + if (denyNode["ip"]) { + YAML::Node ip = denyNode["ip"]; + if (!ip.IsNull()) { + if (ip.IsSequence()) { + // Do IP Deny processing + for (std::size_t i = 0; i < ip.size(); i++) { + IpAddr min, max; + ats_ip_range_parse(std::string_view{ip[i].as()}, min, max); + deny_ip_map.fill(min, max, nullptr); + TSDebug(PLUGIN_NAME, "loading ip: valid: %d, fam %d ", min.isValid(), min.family()); + } + } else { + TSDebug(PLUGIN_NAME, "Invalid IP deny list yaml"); + } + } + } + } catch (const YAML::Exception &e) { + TSDebug("YAML::Exception %s when parsing YAML config file ip deny list for maxmind", e.what()); + return false; + } + + if (denyNode["regex"]) { + YAML::Node regex = denyNode["regex"]; + parseregex(regex, false); + } + +#if 0 + std::unordered_map::iterator cursor; + TSDebug(PLUGIN_NAME, "Deny Country List:"); + for (cursor = allow_country.begin(); cursor != allow_country.end(); cursor++) { + TSDebug(PLUGIN_NAME, "%s:%d", cursor->first.c_str(), cursor->second); + } +#endif + + return true; +} + +// Parse the allow list country codes and IPs +bool +Acl::loadallow(YAML::Node allowNode) +{ + if (!allowNode) { + TSDebug(PLUGIN_NAME, "No Allow rules set"); + return false; + } + if (allowNode.IsNull()) { + TSDebug(PLUGIN_NAME, "Allow rules are NULL"); + return false; + } + +#if 0 + // Test junk + for (YAML::const_iterator it = allowNode.begin(); it != allowNode.end(); ++it) { + const std::string &name = it->first.as(); + YAML::NodeType::value type = it->second.Type(); + TSDebug(PLUGIN_NAME, "name: %s, value: %d", name.c_str(), type); + } +#endif + + // Load Allowable Country codes + try { + if (allowNode["country"]) { + YAML::Node country = allowNode["country"]; + if (!country.IsNull()) { + if (country.IsSequence()) { + for (std::size_t i = 0; i < country.size(); i++) { + allow_country.insert_or_assign(country[i].as(), true); + } + + } else { + TSDebug(PLUGIN_NAME, "Invalid country code allow list yaml"); + } + } + } + } catch (const YAML::Exception &e) { + TSDebug("YAML::Exception %s when parsing YAML config file country code allow list for maxmind", e.what()); + return false; + } + + // Load Allowable IPs + try { + if (allowNode["ip"]) { + YAML::Node ip = allowNode["ip"]; + if (!ip.IsNull()) { + if (ip.IsSequence()) { + // Do IP Allow processing + for (std::size_t i = 0; i < ip.size(); i++) { + IpAddr min, max; + ats_ip_range_parse(std::string_view{ip[i].as()}, min, max); + allow_ip_map.fill(min, max, nullptr); + TSDebug(PLUGIN_NAME, "loading ip: valid: %d, fam %d ", min.isValid(), min.family()); + } + } else { + TSDebug(PLUGIN_NAME, "Invalid IP allow list yaml"); + } + } + } + } catch (const YAML::Exception &e) { + TSDebug("YAML::Exception %s when parsing YAML config file ip allow list for maxmind", e.what()); + return false; + } + + if (allowNode["regex"]) { + YAML::Node regex = allowNode["regex"]; + parseregex(regex, true); + } + +#if 0 + std::unordered_map::iterator cursor; + TSDebug(PLUGIN_NAME, "Allow Country List:"); + for (cursor = allow_country.begin(); cursor != allow_country.end(); cursor++) { + TSDebug(PLUGIN_NAME, "%s:%d", cursor->first.c_str(), cursor->second); + } +#endif + + return true; +} + +void +Acl::parseregex(YAML::Node regex, bool allow) +{ + try { + if (!regex.IsNull()) { + if (regex.IsSequence()) { + // Parse each country-regex pair + for (std::size_t i = 0; i < regex.size(); i++) { + plugin_regex temp; + auto temprule = regex[i].as>(); + temp._regex_s = temprule.back(); + const char *error; + int erroffset; + temp._rex = pcre_compile(temp._regex_s.c_str(), 0, &error, &erroffset, nullptr); + + // Compile the regex for this set of countries + if (nullptr != temp._rex) { + temp._extra = pcre_study(temp._rex, 0, &error); + if ((nullptr == temp._extra) && error && (*error != 0)) { + TSError("[%s] Failed to study regular expression in %s:%s", PLUGIN_NAME, temp._regex_s.c_str(), error); + return; + } + } else { + TSError("[%s] Failed to compile regular expression in %s: %s", PLUGIN_NAME, temp._regex_s.c_str(), error); + return; + } + + for (std::size_t y = 0; y < temprule.size() - 1; y++) { + TSDebug(PLUGIN_NAME, "Adding regex: %s, for country: %s", temp._regex_s.c_str(), regex[i][y].as().c_str()); + if (allow) { + allow_regex[regex[i][y].as()].push_back(temp); + } else { + deny_regex[regex[i][y].as()].push_back(temp); + } + } + } + } + } + } catch (const YAML::Exception &e) { + TSDebug("YAML::Exception %s when parsing YAML config file regex allow list for maxmind", e.what()); + return; + } +} + +void +Acl::loadhtml(YAML::Node htmlNode) +{ + std::string htmlname, htmlloc; + std::ifstream f; + + if (!htmlNode) { + TSDebug(PLUGIN_NAME, "No html field set"); + return; + } + + if (htmlNode.IsNull()) { + TSDebug(PLUGIN_NAME, "Html field not set"); + return; + } + + htmlname = htmlNode.as(); + if (htmlname[0] != '/') { + htmlloc = TSConfigDirGet(); + htmlloc += "/"; + htmlloc.append(htmlname); + } else { + htmlloc.assign(htmlname); + } + + f.open(htmlloc, std::ios::in); + if (f.is_open()) { + _html.append(std::istreambuf_iterator(f), std::istreambuf_iterator()); + f.close(); + TSDebug(PLUGIN_NAME, "Loaded HTML from %s", htmlloc.c_str()); + } else { + TSError("[%s] Unable to open HTML file %s", PLUGIN_NAME, htmlloc.c_str()); + } +} +/////////////////////////////////////////////////////////////////////////////// +// Load the maxmind database from the config parameter +bool +Acl::loaddb(YAML::Node dbNode) +{ + std::string dbloc, dbname; + + if (!dbNode) { + TSDebug(PLUGIN_NAME, "No Database field set"); + return false; + } + if (dbNode.IsNull()) { + TSDebug(PLUGIN_NAME, "Database file not set"); + return false; + } + dbname = dbNode.as(); + if (dbname[0] != '/') { + dbloc = TSConfigDirGet(); + dbloc += "/"; + dbloc.append(dbname); + } else { + dbloc.assign(dbname); + } + + // Make sure we close any previously opened DBs in case this is a reload + if (plugin_state.db_loaded) { + MMDB_close(&_mmdb); + } + + int status = MMDB_open(dbloc.c_str(), MMDB_MODE_MMAP, &_mmdb); + if (MMDB_SUCCESS != status) { + TSDebug(PLUGIN_NAME, "Cant open DB %s - %s", dbloc.c_str(), MMDB_strerror(status)); + return false; + } + + plugin_state.db_loaded = true; + TSDebug(PLUGIN_NAME, "Initialized MMDB with %s", dbloc.c_str()); + return true; +} + +bool +Acl::eval(TSRemapRequestInfo *rri, TSHttpTxn txnp) +{ + bool ret = default_allow; + int mmdb_error; + MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&_mmdb, TSHttpTxnClientAddrGet(txnp), &mmdb_error); + + if (MMDB_SUCCESS != mmdb_error) { + TSDebug(PLUGIN_NAME, "Error during sockaddr lookup: %s", MMDB_strerror(mmdb_error)); + ret = false; + return ret; + } + + MMDB_entry_data_list_s *entry_data_list = nullptr; + if (result.found_entry) { + int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list); + if (MMDB_SUCCESS != status) { + TSDebug(PLUGIN_NAME, "Error looking up entry data: %s", MMDB_strerror(status)); + ret = false; + return ret; + } + + if (NULL != entry_data_list) { + // This is useful to be able to dump out a full record of a + // mmdb entry for debug. Enabling can help if you want to figure + // out how to add new fields +#if 0 + // Block of test stuff to dump output, remove later + char buffer[4096]; + FILE *temp = fmemopen(&buffer[0], 4096, "wb+"); + int status = MMDB_dump_entry_data_list(temp, entry_data_list, 0); + fflush(temp); + TSDebug(PLUGIN_NAME, "Entry: %s, status: %s, type: %d", buffer, MMDB_strerror(status), entry_data_list->entry_data.type); +#endif + + MMDB_entry_data_s entry_data; + int path_len = 0; + const char *path = nullptr; + if (!allow_regex.empty() || !deny_regex.empty()) { + path = TSUrlPathGet(rri->requestBufp, rri->requestUrl, &path_len); + } + // Test for country code + if (!allow_country.empty() || !allow_regex.empty() || !deny_regex.empty()) { + status = MMDB_get_value(&result.entry, &entry_data, "country", "iso_code", NULL); + if (MMDB_SUCCESS != status) { + TSDebug(PLUGIN_NAME, "err on get country code value: %s", MMDB_strerror(status)); + return false; + } + if (entry_data.has_data) { + ret = eval_country(&entry_data, path, path_len); + } + } else { + // Country map is empty as well as regexes, use our default rejection + ret = default_allow; + } + } + } else { + TSDebug(PLUGIN_NAME, "No Country Code entry for this IP was found"); + ret = false; + } + + // Test for allowable IPs based on our lists + switch (eval_ip(TSHttpTxnClientAddrGet(txnp))) { + case ALLOW_IP: + TSDebug(PLUGIN_NAME, "Saw explicit allow of this IP"); + ret = true; + break; + case DENY_IP: + TSDebug(PLUGIN_NAME, "Saw explicit deny of this IP"); + ret = false; + break; + case UNKNOWN_IP: + TSDebug(PLUGIN_NAME, "Unknown IP, following default from ruleset: %d", ret); + break; + default: + TSDebug(PLUGIN_NAME, "Unknown client addr ip state, should not get here"); + ret = false; + break; + } + + if (NULL != entry_data_list) { + MMDB_free_entry_data_list(entry_data_list); + } + + return ret; +} + +/////////////////////////////////////////////////////////////////////////////// +// Returns true if entry data contains an +// allowable country code from our map. +// False otherwise +bool +Acl::eval_country(MMDB_entry_data_s *entry_data, const char *path, int path_len) +{ + bool ret = false; + bool allow = default_allow; + char *output = NULL; + output = (char *)malloc((sizeof(char) * entry_data->data_size)); + strncpy(output, entry_data->utf8_string, entry_data->data_size); + TSDebug(PLUGIN_NAME, "This IP Country Code: %s", output); + auto exists = allow_country.count(output); + + // If the country exists in our map then set its allow value here + // Otherwise we will use our default value + if (exists) { + allow = allow_country[output]; + } + + if (allow) { + TSDebug(PLUGIN_NAME, "Found country code of IP in allow list or allow by default"); + ret = true; + } + + if (nullptr != path && 0 != path_len) { + if (!allow_regex[output].empty()) { + for (auto &i : allow_regex[output]) { + if (PCRE_ERROR_NOMATCH != pcre_exec(i._rex, i._extra, path, path_len, 0, PCRE_NOTEMPTY, nullptr, 0)) { + TSDebug(PLUGIN_NAME, "Got a regex allow hit on regex: %s, country: %s", i._regex_s.c_str(), output); + ret = true; + } + } + } + if (!deny_regex[output].empty()) { + for (auto &i : deny_regex[output]) { + if (PCRE_ERROR_NOMATCH != pcre_exec(i._rex, i._extra, path, path_len, 0, PCRE_NOTEMPTY, nullptr, 0)) { + TSDebug(PLUGIN_NAME, "Got a regex deny hit on regex: %s, country: %s", i._regex_s.c_str(), output); + ret = false; + } + } + } + } + + free(output); + return ret; +} + +/////////////////////////////////////////////////////////////////////////////// +// Returns enum based on current client: +// ALLOW_IP if IP is in the allow list +// DENY_IP if IP is in the deny list +// UNKNOWN_IP if it does not exist in either, this is then used to determine +// action based on the default allow action +ipstate +Acl::eval_ip(const sockaddr *sock) +{ +#if 0 + for (auto &spot : allow_ip_map) { + char text[INET6_ADDRSTRLEN]; + TSDebug(PLUGIN_NAME, "IP: %s", ats_ip_ntop(spot.min(), text, sizeof text)); + if (0 != ats_ip_addr_cmp(spot.min(), spot.max())) { + TSDebug(PLUGIN_NAME, "stuff: %s", ats_ip_ntop(spot.max(), text, sizeof text)); + } + } +#endif + + if (allow_ip_map.contains(sock, nullptr)) { + // Allow map has this ip, we know we want to allow it + return ALLOW_IP; + } + + if (deny_ip_map.contains(sock, nullptr)) { + // Deny map has this ip, explicitly deny + return DENY_IP; + } + + return UNKNOWN_IP; +} diff --git a/plugins/experimental/maxmind_acl/mmdb.h b/plugins/experimental/maxmind_acl/mmdb.h new file mode 100644 index 00000000000..b7d16ecffa3 --- /dev/null +++ b/plugins/experimental/maxmind_acl/mmdb.h @@ -0,0 +1,120 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +//#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tscore/IpMap.h" + +#ifdef HAVE_PCRE_PCRE_H +#include +#else +#include +#endif + +#define PLUGIN_NAME "maxmind_acl" +#define CONFIG_TMOUT 60000 + +typedef struct { + std::string config_file; + time_t last_load; + bool db_loaded; +} plugin_state_t; + +typedef struct { + std::string _regex_s; + pcre *_rex; + pcre_extra *_extra; +} plugin_regex; + +typedef enum { ALLOW_IP, DENY_IP, UNKNOWN_IP } ipstate; + +// Base class for all ACLs +class Acl +{ +public: + Acl() {} + ~Acl() + { + if (plugin_state.db_loaded) { + MMDB_close(&_mmdb); + } + } + + bool eval(TSRemapRequestInfo *rri, TSHttpTxn txnp); + bool init(char *filename); + plugin_state_t * + get_state() + { + return &plugin_state; + } + + void + send_html(TSHttpTxn txnp) const + { + if (_html.size() > 0) { + char *msg = TSstrdup(_html.c_str()); + + TSHttpTxnErrorBodySet(txnp, msg, _html.size(), nullptr); // Defaults to text/html + } + } + +protected: + // Class members + YAML::Node _config; + MMDB_s _mmdb; + std::string _html; + std::unordered_map allow_country; + + std::unordered_map> allow_regex; + std::unordered_map> deny_regex; + + IpMap allow_ip_map; + IpMap deny_ip_map; + + // Do we want to allow by default or not? Useful + // for deny only rules + bool default_allow = false; + + plugin_state_t plugin_state = {"", 0, false}; + + bool loaddb(YAML::Node dbNode); + bool loadallow(YAML::Node allowNode); + bool loaddeny(YAML::Node denyNode); + void loadhtml(YAML::Node htmlNode); + bool eval_country(MMDB_entry_data_s *entry_data, const char *path, int path_len); + void parseregex(YAML::Node regex, bool allow); + ipstate eval_ip(const sockaddr *sock); +}; From 6db7684e4b2f9a084a90a484070f53b0189d26a0 Mon Sep 17 00:00:00 2001 From: Evan Zelkowitz Date: Fri, 10 Jul 2020 15:47:27 +0000 Subject: [PATCH 2/5] Change config file location string to be const to match up all the way down and avoid strdup --- plugins/experimental/maxmind_acl/maxmind_acl.cc | 5 +---- plugins/experimental/maxmind_acl/mmdb.cc | 2 +- plugins/experimental/maxmind_acl/mmdb.h | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/plugins/experimental/maxmind_acl/maxmind_acl.cc b/plugins/experimental/maxmind_acl/maxmind_acl.cc index bb72e72266b..f51c3c2065b 100644 --- a/plugins/experimental/maxmind_acl/maxmind_acl.cc +++ b/plugins/experimental/maxmind_acl/maxmind_acl.cc @@ -29,10 +29,7 @@ config_handler(TSCont cont, TSEvent event, void *edata) TSDebug(PLUGIN_NAME, "In config Handler"); Acl *a = static_cast(TSContDataGet(cont)); - // strdup for const string return - char *config = strdup(a->get_state()->config_file.c_str()); - a->init(config); - free(config); + a->init(a->get_state()->config_file.c_str()); TSMutexUnlock(mutex); // Don't reschedule for TS_EVENT_MGMT_UPDATE diff --git a/plugins/experimental/maxmind_acl/mmdb.cc b/plugins/experimental/maxmind_acl/mmdb.cc index 8b1b48a0cfc..88d12bd3ea1 100644 --- a/plugins/experimental/maxmind_acl/mmdb.cc +++ b/plugins/experimental/maxmind_acl/mmdb.cc @@ -23,7 +23,7 @@ // check for basics // Clear out any existing data since this may be a reload bool -Acl::init(char *filename) +Acl::init(char const *filename) { std::string configloc; struct stat s; diff --git a/plugins/experimental/maxmind_acl/mmdb.h b/plugins/experimental/maxmind_acl/mmdb.h index b7d16ecffa3..32a5d87fabd 100644 --- a/plugins/experimental/maxmind_acl/mmdb.h +++ b/plugins/experimental/maxmind_acl/mmdb.h @@ -74,7 +74,7 @@ class Acl } bool eval(TSRemapRequestInfo *rri, TSHttpTxn txnp); - bool init(char *filename); + bool init(char const *filename); plugin_state_t * get_state() { From 384d1925f73fd64516116d2902e095cc459a390f Mon Sep 17 00:00:00 2001 From: Evan Zelkowitz Date: Fri, 10 Jul 2020 23:46:33 +0000 Subject: [PATCH 3/5] Remove config reloading, since this is a remap plugin and there is not a reliable way to do config reloads for them based on current mechanisms and triggering --- .../experimental/maxmind_acl/maxmind_acl.cc | 27 ----- plugins/experimental/maxmind_acl/mmdb.cc | 105 ++++++++---------- plugins/experimental/maxmind_acl/mmdb.h | 10 +- 3 files changed, 51 insertions(+), 91 deletions(-) diff --git a/plugins/experimental/maxmind_acl/maxmind_acl.cc b/plugins/experimental/maxmind_acl/maxmind_acl.cc index f51c3c2065b..b3e4a355e09 100644 --- a/plugins/experimental/maxmind_acl/maxmind_acl.cc +++ b/plugins/experimental/maxmind_acl/maxmind_acl.cc @@ -18,27 +18,6 @@ #include "mmdb.h" -static int -config_handler(TSCont cont, TSEvent event, void *edata) -{ - TSMutex mutex; - - mutex = TSContMutexGet(cont); - TSMutexLock(mutex); - - TSDebug(PLUGIN_NAME, "In config Handler"); - Acl *a = static_cast(TSContDataGet(cont)); - - a->init(a->get_state()->config_file.c_str()); - TSMutexUnlock(mutex); - - // Don't reschedule for TS_EVENT_MGMT_UPDATE - if (event == TS_EVENT_TIMEOUT) { - TSContScheduleOnPool(cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); - } - return 0; -} - /////////////////////////////////////////////////////////////////////////////// // Initialize the plugin as a remap plugin. // @@ -63,8 +42,6 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) TSReturnCode TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int /* errbuf_size */) { - TSCont config_cont; - if (argc < 3) { TSError("[%s] Unable to create remap instance, missing configuration file", PLUGIN_NAME); return TS_ERROR; @@ -77,10 +54,6 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int / return TS_ERROR; } - config_cont = TSContCreate(config_handler, TSMutexCreate()); - TSContDataSet(config_cont, static_cast(a)); - TSMgmtUpdateRegister(config_cont, PLUGIN_NAME); - TSDebug(PLUGIN_NAME, "created remap instance with configuration %s", argv[2]); return TS_SUCCESS; } diff --git a/plugins/experimental/maxmind_acl/mmdb.cc b/plugins/experimental/maxmind_acl/mmdb.cc index 88d12bd3ea1..caf3e51a0ab 100644 --- a/plugins/experimental/maxmind_acl/mmdb.cc +++ b/plugins/experimental/maxmind_acl/mmdb.cc @@ -45,23 +45,20 @@ Acl::init(char const *filename) return status; } - if ((plugin_state.last_load == 0) || (s.st_mtime > plugin_state.last_load)) { - TSDebug(PLUGIN_NAME, "Loading config %s, lastload: %ld, file time: %ld", configloc.c_str(), plugin_state.last_load, s.st_mtime); - - try { - _config = YAML::LoadFile(configloc.c_str()); + try { + _config = YAML::LoadFile(configloc.c_str()); - if (_config.IsNull()) { - TSDebug(PLUGIN_NAME, "Config file not found or unreadable"); - return status; - } - if (!_config["maxmind"]) { - TSDebug(PLUGIN_NAME, "Config file not in maxmind namespace"); - return status; - } + if (_config.IsNull()) { + TSDebug(PLUGIN_NAME, "Config file not found or unreadable"); + return status; + } + if (!_config["maxmind"]) { + TSDebug(PLUGIN_NAME, "Config file not in maxmind namespace"); + return status; + } - // Get our root maxmind node - maxmind = _config["maxmind"]; + // Get our root maxmind node + maxmind = _config["maxmind"]; #if 0 // Test junk for (YAML::const_iterator it = maxmind.begin(); it != maxmind.end(); ++it) { @@ -70,52 +67,48 @@ Acl::init(char const *filename) TSDebug(PLUGIN_NAME, "name: %s, value: %d", name.c_str(), type); } #endif - } catch (const YAML::Exception &e) { - TSError("YAML::Exception %s when parsing YAML config file %s for maxmind", e.what(), configloc.c_str()); - return status; - } + } catch (const YAML::Exception &e) { + TSError("YAML::Exception %s when parsing YAML config file %s for maxmind", e.what(), configloc.c_str()); + return status; + } - // Find our database name and convert to full path as needed - status = loaddb(maxmind["database"]); + // Find our database name and convert to full path as needed + status = loaddb(maxmind["database"]); - if (!status) { - TSDebug(PLUGIN_NAME, "Failed to load MaxMind Database"); - return status; - } + if (!status) { + TSDebug(PLUGIN_NAME, "Failed to load MaxMind Database"); + return status; + } - // Clear out existing data, these may no longer exist in a new config and so we - // dont want old ones left behind - allow_country.clear(); - allow_ip_map.clear(); - deny_ip_map.clear(); - allow_regex.clear(); - deny_regex.clear(); - _html.clear(); - default_allow = false; - - if (loadallow(maxmind["allow"])) { - TSDebug(PLUGIN_NAME, "Loaded Allow ruleset"); - status = true; - } else { - // We have no proper allow ruleset - // setting to allow by default to only apply deny rules - default_allow = true; - } + // Clear out existing data, these may no longer exist in a new config and so we + // dont want old ones left behind + allow_country.clear(); + allow_ip_map.clear(); + deny_ip_map.clear(); + allow_regex.clear(); + deny_regex.clear(); + _html.clear(); + default_allow = false; - if (loaddeny(maxmind["deny"])) { - TSDebug(PLUGIN_NAME, "Loaded Deny ruleset"); - status = true; - } + if (loadallow(maxmind["allow"])) { + TSDebug(PLUGIN_NAME, "Loaded Allow ruleset"); + status = true; + } else { + // We have no proper allow ruleset + // setting to allow by default to only apply deny rules + default_allow = true; + } - loadhtml(maxmind["html"]); + if (loaddeny(maxmind["deny"])) { + TSDebug(PLUGIN_NAME, "Loaded Deny ruleset"); + status = true; + } - if (!status) { - TSDebug(PLUGIN_NAME, "Failed to load any rulesets, none specified"); - status = false; - } + loadhtml(maxmind["html"]); - plugin_state.config_file = configloc; - plugin_state.last_load = time(NULL); + if (!status) { + TSDebug(PLUGIN_NAME, "Failed to load any rulesets, none specified"); + status = false; } return status; @@ -386,7 +379,7 @@ Acl::loaddb(YAML::Node dbNode) } // Make sure we close any previously opened DBs in case this is a reload - if (plugin_state.db_loaded) { + if (db_loaded) { MMDB_close(&_mmdb); } @@ -396,7 +389,7 @@ Acl::loaddb(YAML::Node dbNode) return false; } - plugin_state.db_loaded = true; + db_loaded = true; TSDebug(PLUGIN_NAME, "Initialized MMDB with %s", dbloc.c_str()); return true; } diff --git a/plugins/experimental/maxmind_acl/mmdb.h b/plugins/experimental/maxmind_acl/mmdb.h index 32a5d87fabd..00e10836a2a 100644 --- a/plugins/experimental/maxmind_acl/mmdb.h +++ b/plugins/experimental/maxmind_acl/mmdb.h @@ -68,18 +68,13 @@ class Acl Acl() {} ~Acl() { - if (plugin_state.db_loaded) { + if (db_loaded) { MMDB_close(&_mmdb); } } bool eval(TSRemapRequestInfo *rri, TSHttpTxn txnp); bool init(char const *filename); - plugin_state_t * - get_state() - { - return &plugin_state; - } void send_html(TSHttpTxn txnp) const @@ -107,8 +102,7 @@ class Acl // Do we want to allow by default or not? Useful // for deny only rules bool default_allow = false; - - plugin_state_t plugin_state = {"", 0, false}; + bool db_loaded = false; bool loaddb(YAML::Node dbNode); bool loadallow(YAML::Node allowNode); From e3058e17162c5cd3595c8a17e11ff59504b6619e Mon Sep 17 00:00:00 2001 From: Evan Zelkowitz Date: Fri, 10 Jul 2020 23:49:18 +0000 Subject: [PATCH 4/5] Remove now unneeded plugin_state struct --- plugins/experimental/maxmind_acl/mmdb.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugins/experimental/maxmind_acl/mmdb.h b/plugins/experimental/maxmind_acl/mmdb.h index 00e10836a2a..828ffe3bff9 100644 --- a/plugins/experimental/maxmind_acl/mmdb.h +++ b/plugins/experimental/maxmind_acl/mmdb.h @@ -47,12 +47,6 @@ #define PLUGIN_NAME "maxmind_acl" #define CONFIG_TMOUT 60000 -typedef struct { - std::string config_file; - time_t last_load; - bool db_loaded; -} plugin_state_t; - typedef struct { std::string _regex_s; pcre *_rex; From c1f36f6199b8dd310e719249898dbc3981950aee Mon Sep 17 00:00:00 2001 From: Evan Zelkowitz Date: Mon, 13 Jul 2020 18:48:37 +0000 Subject: [PATCH 5/5] Add docs note about needing to modify or touch remap.config to get a plugin to reload a new config --- doc/admin-guide/plugins/maxmind_acl.en.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/admin-guide/plugins/maxmind_acl.en.rst b/doc/admin-guide/plugins/maxmind_acl.en.rst index e783b0bec1e..2b13ab58223 100644 --- a/doc/admin-guide/plugins/maxmind_acl.en.rst +++ b/doc/admin-guide/plugins/maxmind_acl.en.rst @@ -54,6 +54,8 @@ An example configuration :: - [US, ".*\\.txt"] # Because these get parsed you must escape the escape of the ``.`` in order to have it be escaped in the regex, resulting in ".*\.txt" - [US, ".*\\.mp3"] +In order to load an updated configuration while ATS is running you will have to touch or modify the remap.config file in order to initiate a plugin reload to pull in any changes. + Rules =====